2024-11-13 00:13:18,838 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 00:13:18,852 main DEBUG Took 0.011099 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 00:13:18,853 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 00:13:18,853 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 00:13:18,854 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 00:13:18,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,868 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 00:13:18,890 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,892 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,893 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,894 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,897 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,898 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,899 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,900 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,901 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,902 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,904 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:13:18,907 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,907 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 00:13:18,909 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:13:18,911 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 00:13:18,913 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 00:13:18,914 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 00:13:18,915 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 00:13:18,916 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 00:13:18,926 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 00:13:18,929 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 00:13:18,931 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 00:13:18,932 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 00:13:18,932 main DEBUG createAppenders(={Console}) 2024-11-13 00:13:18,933 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 00:13:18,934 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 00:13:18,934 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 00:13:18,935 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 00:13:18,935 main DEBUG OutputStream closed 2024-11-13 00:13:18,936 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 00:13:18,936 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 00:13:18,936 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 00:13:19,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 00:13:19,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 00:13:19,042 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 00:13:19,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 00:13:19,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 00:13:19,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 00:13:19,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 00:13:19,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 00:13:19,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 00:13:19,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 00:13:19,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 00:13:19,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 00:13:19,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 00:13:19,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 00:13:19,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 00:13:19,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 00:13:19,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 00:13:19,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 00:13:19,050 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 00:13:19,050 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 00:13:19,050 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 00:13:19,051 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T00:13:19,409 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de 2024-11-13 00:13:19,414 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 00:13:19,414 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T00:13:19,427 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-13T00:13:19,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T00:13:19,482 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948, deleteOnExit=true 2024-11-13T00:13:19,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T00:13:19,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/test.cache.data in system properties and HBase conf 2024-11-13T00:13:19,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T00:13:19,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/hadoop.log.dir in system properties and HBase conf 2024-11-13T00:13:19,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T00:13:19,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T00:13:19,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T00:13:19,617 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T00:13:19,744 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T00:13:19,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T00:13:19,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T00:13:19,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T00:13:19,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T00:13:19,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T00:13:19,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T00:13:19,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T00:13:19,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T00:13:19,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T00:13:19,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/nfs.dump.dir in system properties and HBase conf 2024-11-13T00:13:19,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/java.io.tmpdir in system properties and HBase conf 2024-11-13T00:13:19,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T00:13:19,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T00:13:19,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T00:13:20,688 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T00:13:20,772 INFO [Time-limited test {}] log.Log(170): Logging initialized @2865ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T00:13:20,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:13:20,938 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T00:13:20,973 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T00:13:20,973 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T00:13:20,975 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T00:13:20,991 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:13:20,994 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/hadoop.log.dir/,AVAILABLE} 2024-11-13T00:13:20,995 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T00:13:21,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/java.io.tmpdir/jetty-localhost-45743-hadoop-hdfs-3_4_1-tests_jar-_-any-15628872484523949694/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T00:13:21,253 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45743} 2024-11-13T00:13:21,253 INFO [Time-limited test {}] server.Server(415): Started @3346ms 2024-11-13T00:13:21,701 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:13:21,710 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T00:13:21,711 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T00:13:21,711 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T00:13:21,711 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T00:13:21,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/hadoop.log.dir/,AVAILABLE} 2024-11-13T00:13:21,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T00:13:21,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/java.io.tmpdir/jetty-localhost-34467-hadoop-hdfs-3_4_1-tests_jar-_-any-12926925461961621932/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:13:21,841 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:34467} 2024-11-13T00:13:21,842 INFO [Time-limited test {}] server.Server(415): Started @3935ms 2024-11-13T00:13:21,897 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T00:13:22,589 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/data/data2/current/BP-1378608587-172.17.0.2-1731456800440/current, will proceed with Du for space computation calculation, 2024-11-13T00:13:22,589 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/data/data1/current/BP-1378608587-172.17.0.2-1731456800440/current, will proceed with Du for space computation calculation, 2024-11-13T00:13:22,660 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T00:13:22,726 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf33a2529540cff16 with lease ID 0xec83d39efeb9fcbe: Processing first storage report for DS-b2329746-bd14-494b-807c-9bf87128923f from datanode DatanodeRegistration(127.0.0.1:45587, datanodeUuid=01441b4e-4e4a-4f75-ac2b-c3de0838f229, infoPort=37623, infoSecurePort=0, ipcPort=41105, storageInfo=lv=-57;cid=testClusterID;nsid=1375938288;c=1731456800440) 2024-11-13T00:13:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf33a2529540cff16 with lease ID 0xec83d39efeb9fcbe: from storage DS-b2329746-bd14-494b-807c-9bf87128923f node DatanodeRegistration(127.0.0.1:45587, datanodeUuid=01441b4e-4e4a-4f75-ac2b-c3de0838f229, infoPort=37623, infoSecurePort=0, ipcPort=41105, storageInfo=lv=-57;cid=testClusterID;nsid=1375938288;c=1731456800440), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T00:13:22,728 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf33a2529540cff16 with lease ID 0xec83d39efeb9fcbe: Processing first storage report for DS-e7cd56ee-3759-4377-a8b2-2c69270051db from datanode DatanodeRegistration(127.0.0.1:45587, datanodeUuid=01441b4e-4e4a-4f75-ac2b-c3de0838f229, infoPort=37623, infoSecurePort=0, ipcPort=41105, storageInfo=lv=-57;cid=testClusterID;nsid=1375938288;c=1731456800440) 2024-11-13T00:13:22,728 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf33a2529540cff16 with lease ID 0xec83d39efeb9fcbe: from storage DS-e7cd56ee-3759-4377-a8b2-2c69270051db node DatanodeRegistration(127.0.0.1:45587, datanodeUuid=01441b4e-4e4a-4f75-ac2b-c3de0838f229, infoPort=37623, infoSecurePort=0, ipcPort=41105, storageInfo=lv=-57;cid=testClusterID;nsid=1375938288;c=1731456800440), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T00:13:22,823 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de 2024-11-13T00:13:22,900 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/zookeeper_0, clientPort=55432, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T00:13:22,913 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55432 2024-11-13T00:13:22,927 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:22,931 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:23,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741825_1001 (size=7) 2024-11-13T00:13:23,590 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 with version=8 2024-11-13T00:13:23,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/hbase-staging 2024-11-13T00:13:23,686 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T00:13:23,948 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cb28580903fb:0 server-side Connection retries=45 2024-11-13T00:13:23,960 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:13:23,961 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T00:13:23,967 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T00:13:23,967 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:13:23,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T00:13:24,145 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T00:13:24,226 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T00:13:24,238 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T00:13:24,243 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T00:13:24,281 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1005 (auto-detected) 2024-11-13T00:13:24,282 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T00:13:24,315 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41869 2024-11-13T00:13:24,344 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41869 connecting to ZooKeeper ensemble=127.0.0.1:55432 2024-11-13T00:13:24,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418690x0, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T00:13:24,383 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41869-0x1010eac0ace0000 connected 2024-11-13T00:13:24,427 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:24,431 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:24,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:13:24,450 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54, hbase.cluster.distributed=false 2024-11-13T00:13:24,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T00:13:24,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41869 2024-11-13T00:13:24,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41869 2024-11-13T00:13:24,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41869 2024-11-13T00:13:24,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41869 2024-11-13T00:13:24,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41869 2024-11-13T00:13:24,636 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cb28580903fb:0 server-side Connection retries=45 2024-11-13T00:13:24,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:13:24,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T00:13:24,638 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T00:13:24,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:13:24,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T00:13:24,641 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T00:13:24,644 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T00:13:24,646 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45273 2024-11-13T00:13:24,648 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45273 connecting to ZooKeeper ensemble=127.0.0.1:55432 2024-11-13T00:13:24,650 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:24,655 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:24,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452730x0, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:13:24,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452730x0, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T00:13:24,675 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45273-0x1010eac0ace0001 connected 2024-11-13T00:13:24,676 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T00:13:24,684 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T00:13:24,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T00:13:24,693 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T00:13:24,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45273 2024-11-13T00:13:24,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45273 2024-11-13T00:13:24,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45273 2024-11-13T00:13:24,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45273 2024-11-13T00:13:24,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45273 2024-11-13T00:13:24,715 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cb28580903fb:41869 2024-11-13T00:13:24,716 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cb28580903fb,41869,1731456803739 2024-11-13T00:13:24,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:13:24,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:13:24,728 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cb28580903fb,41869,1731456803739 2024-11-13T00:13:24,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T00:13:24,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:24,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:24,754 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T00:13:24,755 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cb28580903fb,41869,1731456803739 from backup master directory 2024-11-13T00:13:24,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cb28580903fb,41869,1731456803739 2024-11-13T00:13:24,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:13:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:13:24,760 WARN [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T00:13:24,760 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cb28580903fb,41869,1731456803739 2024-11-13T00:13:24,762 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T00:13:24,764 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T00:13:24,833 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/hbase.id] with ID: 2df54b55-b6b1-4607-8c19-554f8a3d512f 2024-11-13T00:13:24,834 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/.tmp/hbase.id 2024-11-13T00:13:24,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741826_1002 (size=42) 2024-11-13T00:13:25,252 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/.tmp/hbase.id]:[hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/hbase.id] 2024-11-13T00:13:25,306 INFO [master/cb28580903fb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:13:25,312 INFO [master/cb28580903fb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T00:13:25,337 INFO [master/cb28580903fb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-13T00:13:25,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:25,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:25,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741827_1003 (size=196) 2024-11-13T00:13:25,778 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:13:25,781 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T00:13:25,797 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:25,802 INFO [master/cb28580903fb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:13:25,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741828_1004 (size=1189) 2024-11-13T00:13:26,254 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store 2024-11-13T00:13:26,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741829_1005 (size=34) 2024-11-13T00:13:26,276 INFO [master/cb28580903fb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T00:13:26,280 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:13:26,281 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T00:13:26,281 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:13:26,282 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:13:26,284 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T00:13:26,284 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:13:26,284 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:13:26,286 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731456806281Disabling compacts and flushes for region at 1731456806281Disabling writes for close at 1731456806284 (+3 ms)Writing region close event to WAL at 1731456806284Closed at 1731456806284 2024-11-13T00:13:26,288 WARN [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/.initializing 2024-11-13T00:13:26,288 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/WALs/cb28580903fb,41869,1731456803739 2024-11-13T00:13:26,299 INFO [master/cb28580903fb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:13:26,316 INFO [master/cb28580903fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C41869%2C1731456803739, suffix=, logDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/WALs/cb28580903fb,41869,1731456803739, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/oldWALs, maxLogs=10 2024-11-13T00:13:26,338 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/WALs/cb28580903fb,41869,1731456803739/cb28580903fb%2C41869%2C1731456803739.1731456806320, exclude list is [], retry=0 2024-11-13T00:13:26,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b2329746-bd14-494b-807c-9bf87128923f,DISK] 2024-11-13T00:13:26,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T00:13:26,391 INFO [master/cb28580903fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/WALs/cb28580903fb,41869,1731456803739/cb28580903fb%2C41869%2C1731456803739.1731456806320 2024-11-13T00:13:26,392 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37623:37623)] 2024-11-13T00:13:26,393 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:13:26,393 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:13:26,398 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,399 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,462 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T00:13:26,465 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:26,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:26,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T00:13:26,471 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:26,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:13:26,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T00:13:26,475 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:26,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:13:26,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T00:13:26,479 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:26,479 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:13:26,480 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,483 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,484 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,489 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,490 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,494 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T00:13:26,498 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:13:26,502 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:13:26,503 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73641351, jitterRate=0.0973416417837143}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T00:13:26,510 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731456806412Initializing all the Stores at 1731456806414 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456806415 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456806415Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456806415Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456806415Cleaning up temporary data from old regions at 1731456806490 (+75 ms)Region opened successfully at 1731456806510 (+20 ms) 2024-11-13T00:13:26,512 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T00:13:26,551 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7847f287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb28580903fb/172.17.0.2:0 2024-11-13T00:13:26,583 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T00:13:26,594 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T00:13:26,594 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T00:13:26,597 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T00:13:26,598 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T00:13:26,603 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T00:13:26,603 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T00:13:26,627 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T00:13:26,636 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T00:13:26,638 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T00:13:26,640 INFO [master/cb28580903fb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T00:13:26,641 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T00:13:26,643 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T00:13:26,645 INFO [master/cb28580903fb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T00:13:26,648 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T00:13:26,650 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T00:13:26,651 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T00:13:26,654 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T00:13:26,671 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T00:13:26,672 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T00:13:26,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T00:13:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T00:13:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:26,679 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cb28580903fb,41869,1731456803739, sessionid=0x1010eac0ace0000, setting cluster-up flag (Was=false) 2024-11-13T00:13:26,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:26,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:26,698 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T00:13:26,699 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cb28580903fb,41869,1731456803739 2024-11-13T00:13:26,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:26,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:26,716 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T00:13:26,718 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cb28580903fb,41869,1731456803739 2024-11-13T00:13:26,724 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T00:13:26,799 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T00:13:26,804 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(746): ClusterId : 2df54b55-b6b1-4607-8c19-554f8a3d512f 2024-11-13T00:13:26,807 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T00:13:26,809 INFO [master/cb28580903fb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T00:13:26,812 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T00:13:26,812 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T00:13:26,815 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T00:13:26,815 DEBUG [RS:0;cb28580903fb:45273 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db46969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb28580903fb/172.17.0.2:0 2024-11-13T00:13:26,816 INFO [master/cb28580903fb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T00:13:26,822 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cb28580903fb,41869,1731456803739 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T00:13:26,828 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:13:26,829 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:13:26,829 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:13:26,829 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:13:26,829 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cb28580903fb:0, corePoolSize=10, maxPoolSize=10 2024-11-13T00:13:26,829 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,829 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cb28580903fb:0, corePoolSize=2, maxPoolSize=2 2024-11-13T00:13:26,830 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,831 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731456836831 2024-11-13T00:13:26,832 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T00:13:26,833 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T00:13:26,834 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cb28580903fb:45273 2024-11-13T00:13:26,834 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T00:13:26,835 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T00:13:26,837 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T00:13:26,837 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T00:13:26,838 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T00:13:26,838 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T00:13:26,839 INFO [RS:0;cb28580903fb:45273 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T00:13:26,839 INFO [RS:0;cb28580903fb:45273 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T00:13:26,839 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T00:13:26,838 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,841 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:26,842 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T00:13:26,842 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T00:13:26,842 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,41869,1731456803739 with port=45273, startcode=1731456804594 2024-11-13T00:13:26,843 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T00:13:26,843 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T00:13:26,845 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T00:13:26,846 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T00:13:26,847 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.large.0-1731456806847,5,FailOnTimeoutGroup] 2024-11-13T00:13:26,849 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.small.0-1731456806847,5,FailOnTimeoutGroup] 2024-11-13T00:13:26,849 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,849 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T00:13:26,850 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,851 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741831_1007 (size=1321) 2024-11-13T00:13:26,855 DEBUG [RS:0;cb28580903fb:45273 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T00:13:26,888 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T00:13:26,896 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41869 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cb28580903fb,45273,1731456804594 2024-11-13T00:13:26,899 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41869 {}] master.ServerManager(517): Registering regionserver=cb28580903fb,45273,1731456804594 2024-11-13T00:13:26,916 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:13:26,917 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45809 2024-11-13T00:13:26,917 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T00:13:26,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T00:13:26,922 DEBUG [RS:0;cb28580903fb:45273 {}] zookeeper.ZKUtil(111): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cb28580903fb,45273,1731456804594 2024-11-13T00:13:26,922 WARN [RS:0;cb28580903fb:45273 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T00:13:26,922 INFO [RS:0;cb28580903fb:45273 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:13:26,922 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594 2024-11-13T00:13:26,924 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cb28580903fb,45273,1731456804594] 2024-11-13T00:13:26,948 INFO [RS:0;cb28580903fb:45273 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T00:13:26,961 INFO [RS:0;cb28580903fb:45273 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T00:13:26,966 INFO [RS:0;cb28580903fb:45273 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T00:13:26,967 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,967 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T00:13:26,973 INFO [RS:0;cb28580903fb:45273 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T00:13:26,975 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,975 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,975 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,975 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,975 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,976 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,976 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0, corePoolSize=2, maxPoolSize=2 2024-11-13T00:13:26,976 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,976 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,976 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,976 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,977 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,977 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:13:26,977 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:13:26,977 DEBUG [RS:0;cb28580903fb:45273 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:13:26,978 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,978 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,978 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,978 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,979 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:26,979 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,45273,1731456804594-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T00:13:26,998 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T00:13:27,000 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,45273,1731456804594-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:27,001 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:27,001 INFO [RS:0;cb28580903fb:45273 {}] regionserver.Replication(171): cb28580903fb,45273,1731456804594 started 2024-11-13T00:13:27,020 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:27,020 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1482): Serving as cb28580903fb,45273,1731456804594, RpcServer on cb28580903fb/172.17.0.2:45273, sessionid=0x1010eac0ace0001 2024-11-13T00:13:27,021 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T00:13:27,021 DEBUG [RS:0;cb28580903fb:45273 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cb28580903fb,45273,1731456804594 2024-11-13T00:13:27,021 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,45273,1731456804594' 2024-11-13T00:13:27,021 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T00:13:27,022 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T00:13:27,023 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T00:13:27,023 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T00:13:27,023 DEBUG [RS:0;cb28580903fb:45273 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cb28580903fb,45273,1731456804594 2024-11-13T00:13:27,024 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,45273,1731456804594' 2024-11-13T00:13:27,024 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T00:13:27,024 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T00:13:27,025 DEBUG [RS:0;cb28580903fb:45273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T00:13:27,025 INFO [RS:0;cb28580903fb:45273 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T00:13:27,025 INFO [RS:0;cb28580903fb:45273 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T00:13:27,130 INFO [RS:0;cb28580903fb:45273 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:13:27,134 INFO [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C45273%2C1731456804594, suffix=, logDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/oldWALs, maxLogs=32 2024-11-13T00:13:27,151 DEBUG [RS:0;cb28580903fb:45273 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594/cb28580903fb%2C45273%2C1731456804594.1731456807136, exclude list is [], retry=0 2024-11-13T00:13:27,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b2329746-bd14-494b-807c-9bf87128923f,DISK] 2024-11-13T00:13:27,159 INFO [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594/cb28580903fb%2C45273%2C1731456804594.1731456807136 2024-11-13T00:13:27,160 DEBUG [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37623:37623)] 2024-11-13T00:13:27,254 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T00:13:27,255 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:13:27,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741833_1009 (size=32) 2024-11-13T00:13:27,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:13:27,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T00:13:27,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T00:13:27,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:27,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:27,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T00:13:27,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T00:13:27,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:27,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:27,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T00:13:27,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T00:13:27,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:27,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:27,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T00:13:27,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T00:13:27,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:27,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:27,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T00:13:27,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740 2024-11-13T00:13:27,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740 2024-11-13T00:13:27,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T00:13:27,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T00:13:27,689 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:13:27,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T00:13:27,695 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:13:27,696 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67320282, jitterRate=0.003150373697280884}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:13:27,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731456807666Initializing all the Stores at 1731456807668 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456807668Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456807669 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456807669Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456807669Cleaning up temporary data from old regions at 1731456807688 (+19 ms)Region opened successfully at 1731456807698 (+10 ms) 2024-11-13T00:13:27,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T00:13:27,699 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T00:13:27,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T00:13:27,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T00:13:27,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T00:13:27,700 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T00:13:27,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731456807698Disabling compacts and flushes for region at 1731456807698Disabling writes for close at 1731456807699 (+1 ms)Writing region close event to WAL at 1731456807700 (+1 ms)Closed at 1731456807700 2024-11-13T00:13:27,703 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T00:13:27,704 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T00:13:27,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T00:13:27,720 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T00:13:27,723 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T00:13:27,876 DEBUG [cb28580903fb:41869 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T00:13:27,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:13:27,895 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cb28580903fb,45273,1731456804594, state=OPENING 2024-11-13T00:13:27,901 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T00:13:27,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:27,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:13:27,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:13:27,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:13:27,905 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T00:13:27,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:13:28,082 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T00:13:28,086 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41947, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T00:13:28,096 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T00:13:28,096 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:13:28,097 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T00:13:28,100 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C45273%2C1731456804594.meta, suffix=.meta, logDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/oldWALs, maxLogs=32 2024-11-13T00:13:28,116 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594/cb28580903fb%2C45273%2C1731456804594.meta.1731456808101.meta, exclude list is [], retry=0 2024-11-13T00:13:28,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b2329746-bd14-494b-807c-9bf87128923f,DISK] 2024-11-13T00:13:28,123 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594/cb28580903fb%2C45273%2C1731456804594.meta.1731456808101.meta 2024-11-13T00:13:28,123 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37623:37623)] 2024-11-13T00:13:28,124 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:13:28,126 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T00:13:28,128 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T00:13:28,133 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T00:13:28,138 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T00:13:28,138 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:13:28,138 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T00:13:28,139 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T00:13:28,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T00:13:28,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T00:13:28,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:28,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:28,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T00:13:28,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T00:13:28,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:28,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:28,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T00:13:28,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T00:13:28,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:28,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:28,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T00:13:28,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T00:13:28,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:28,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:13:28,153 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T00:13:28,154 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740 2024-11-13T00:13:28,156 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740 2024-11-13T00:13:28,158 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T00:13:28,159 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T00:13:28,159 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:13:28,162 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T00:13:28,163 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74293022, jitterRate=0.10705229640007019}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:13:28,164 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T00:13:28,165 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731456808139Writing region info on filesystem at 1731456808139Initializing all the Stores at 1731456808141 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456808141Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456808141Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456808141Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731456808141Cleaning up temporary data from old regions at 1731456808159 (+18 ms)Running coprocessor post-open hooks at 1731456808164 (+5 ms)Region opened successfully at 1731456808165 (+1 ms) 2024-11-13T00:13:28,172 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731456808073 2024-11-13T00:13:28,184 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T00:13:28,185 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T00:13:28,186 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:13:28,188 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cb28580903fb,45273,1731456804594, state=OPEN 2024-11-13T00:13:28,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T00:13:28,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T00:13:28,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:13:28,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:13:28,196 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:28,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T00:13:28,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cb28580903fb,45273,1731456804594 in 289 msec 2024-11-13T00:13:28,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T00:13:28,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 492 msec 2024-11-13T00:13:28,208 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T00:13:28,208 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T00:13:28,230 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:28,232 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:28,252 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:28,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34877, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:28,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5190 sec 2024-11-13T00:13:28,274 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731456808274, completionTime=-1 2024-11-13T00:13:28,276 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T00:13:28,277 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T00:13:28,305 INFO [master/cb28580903fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T00:13:28,305 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731456868305 2024-11-13T00:13:28,305 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731456928305 2024-11-13T00:13:28,305 INFO [master/cb28580903fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-13T00:13:28,307 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,41869,1731456803739-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,307 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,41869,1731456803739-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,308 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,41869,1731456803739-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,309 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cb28580903fb:41869, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,309 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,310 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,316 DEBUG [master/cb28580903fb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T00:13:28,341 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.581sec 2024-11-13T00:13:28,342 INFO [master/cb28580903fb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T00:13:28,344 INFO [master/cb28580903fb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T00:13:28,345 INFO [master/cb28580903fb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T00:13:28,345 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T00:13:28,345 INFO [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T00:13:28,346 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,41869,1731456803739-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T00:13:28,346 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,41869,1731456803739-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T00:13:28,354 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T00:13:28,355 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T00:13:28,356 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,41869,1731456803739-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:13:28,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:28,416 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T00:13:28,417 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T00:13:28,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:28,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:28,431 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7607adcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:28,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:28,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:28,442 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:28,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:28,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cb28580903fb,41869,1731456803739 2024-11-13T00:13:28,486 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=2124 2024-11-13T00:13:28,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:13:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:13:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-13T00:13:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:13:28,506 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:13:28,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:13:28,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:13:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T00:13:28,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:13:28,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T00:13:28,526 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:28,528 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:13:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:13:28,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741835_1011 (size=1061) 2024-11-13T00:13:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:13:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:13:28,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:13:28,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741836_1012 (size=53) 2024-11-13T00:13:28,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:13:28,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 883bdd7db86faeb1b8a9722c1fd955c1, disabling compactions & flushes 2024-11-13T00:13:28,953 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:28,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:28,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. after waiting 0 ms 2024-11-13T00:13:28,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:28,953 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:28,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 883bdd7db86faeb1b8a9722c1fd955c1: Waiting for close lock at 1731456808953Disabling compacts and flushes for region at 1731456808953Disabling writes for close at 1731456808953Writing region close event to WAL at 1731456808953Closed at 1731456808953 2024-11-13T00:13:28,956 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:13:28,963 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731456808956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731456808956"}]},"ts":"1731456808956"} 2024-11-13T00:13:28,967 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:13:28,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:13:28,972 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456808969"}]},"ts":"1731456808969"} 2024-11-13T00:13:28,976 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T00:13:28,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, ASSIGN}] 2024-11-13T00:13:28,980 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, ASSIGN 2024-11-13T00:13:28,982 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, ASSIGN; state=OFFLINE, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=false 2024-11-13T00:13:29,134 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=883bdd7db86faeb1b8a9722c1fd955c1, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:13:29,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, ASSIGN because future has completed 2024-11-13T00:13:29,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:13:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:13:29,301 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:29,301 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:13:29,302 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,302 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:13:29,302 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,302 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,305 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,309 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:13:29,309 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883bdd7db86faeb1b8a9722c1fd955c1 columnFamilyName A 2024-11-13T00:13:29,309 DEBUG [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:29,311 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.HStore(327): Store=883bdd7db86faeb1b8a9722c1fd955c1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:13:29,311 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,313 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:13:29,314 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883bdd7db86faeb1b8a9722c1fd955c1 columnFamilyName B 2024-11-13T00:13:29,314 DEBUG [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:29,315 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.HStore(327): Store=883bdd7db86faeb1b8a9722c1fd955c1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:13:29,315 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,317 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:13:29,318 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883bdd7db86faeb1b8a9722c1fd955c1 columnFamilyName C 2024-11-13T00:13:29,318 DEBUG [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:13:29,319 INFO [StoreOpener-883bdd7db86faeb1b8a9722c1fd955c1-1 {}] regionserver.HStore(327): Store=883bdd7db86faeb1b8a9722c1fd955c1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:13:29,319 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:29,319 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,321 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,322 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,324 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,324 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,325 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:13:29,327 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,331 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:13:29,332 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 883bdd7db86faeb1b8a9722c1fd955c1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72761109, jitterRate=0.08422501385211945}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:13:29,332 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,333 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 883bdd7db86faeb1b8a9722c1fd955c1: Running coprocessor pre-open hook at 1731456809302Writing region info on filesystem at 1731456809302Initializing all the Stores at 1731456809304 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456809304Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456809304Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456809304Cleaning up temporary data from old regions at 1731456809324 (+20 ms)Running coprocessor post-open hooks at 1731456809332 (+8 ms)Region opened successfully at 1731456809333 (+1 ms) 2024-11-13T00:13:29,336 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., pid=6, masterSystemTime=1731456809294 2024-11-13T00:13:29,340 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:29,340 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:29,341 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=883bdd7db86faeb1b8a9722c1fd955c1, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:13:29,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:13:29,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T00:13:29,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 in 206 msec 2024-11-13T00:13:29,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T00:13:29,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, ASSIGN in 372 msec 2024-11-13T00:13:29,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:13:29,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456809355"}]},"ts":"1731456809355"} 2024-11-13T00:13:29,358 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T00:13:29,360 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:13:29,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 844 msec 2024-11-13T00:13:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:13:29,667 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:29,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c32821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d29b8ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,680 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,687 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:13:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:13:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59e86fe5 2024-11-13T00:13:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:13:29,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:13:29,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,710 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,718 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbd72e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,721 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69746154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2605ee65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,732 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,734 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5346dc42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,737 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49742bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,741 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,743 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:13:29,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:13:29,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:13:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:13:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:13:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:13:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:13:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:29,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:13:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:13:29,753 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,753 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,753 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,753 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,755 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,755 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,758 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,763 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,802 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:13:29,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:13:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:13:29,813 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:13:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T00:13:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:29,832 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:29,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T00:13:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:13:29,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:13:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:29,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T00:13:30,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T00:13:30,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:30,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456870032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456870032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456870038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456870048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456870043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f77190f58459493dbfd8a75f25f84526 is 50, key is test_row_0/A:col10/1731456809904/Put/seqid=0 2024-11-13T00:13:30,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741837_1013 (size=9657) 2024-11-13T00:13:30,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f77190f58459493dbfd8a75f25f84526 2024-11-13T00:13:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T00:13:30,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T00:13:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456870218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456870221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456870221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456870224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456870219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9fa32bfaccac47438ae2ff3a44d2a1a4 is 50, key is test_row_0/B:col10/1731456809904/Put/seqid=0 2024-11-13T00:13:30,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741838_1014 (size=9657) 2024-11-13T00:13:30,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9fa32bfaccac47438ae2ff3a44d2a1a4 2024-11-13T00:13:30,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/1e821e16141e410ab2e566212519ec7d is 50, key is test_row_0/C:col10/1731456809904/Put/seqid=0 2024-11-13T00:13:30,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T00:13:30,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741839_1015 (size=9657) 2024-11-13T00:13:30,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:30,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/1e821e16141e410ab2e566212519ec7d 2024-11-13T00:13:30,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f77190f58459493dbfd8a75f25f84526 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f77190f58459493dbfd8a75f25f84526 2024-11-13T00:13:30,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456870438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f77190f58459493dbfd8a75f25f84526, entries=100, sequenceid=13, filesize=9.4 K 2024-11-13T00:13:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456870438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456870439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456870447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9fa32bfaccac47438ae2ff3a44d2a1a4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9fa32bfaccac47438ae2ff3a44d2a1a4 2024-11-13T00:13:30,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456870447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9fa32bfaccac47438ae2ff3a44d2a1a4, entries=100, sequenceid=13, filesize=9.4 K 2024-11-13T00:13:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T00:13:30,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/1e821e16141e410ab2e566212519ec7d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/1e821e16141e410ab2e566212519ec7d 2024-11-13T00:13:30,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/1e821e16141e410ab2e566212519ec7d, entries=100, sequenceid=13, filesize=9.4 K 2024-11-13T00:13:30,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 883bdd7db86faeb1b8a9722c1fd955c1 in 602ms, sequenceid=13, compaction requested=false 2024-11-13T00:13:30,524 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T00:13:30,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:30,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T00:13:30,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:30,569 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:13:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/dc33743e837a411b811235a6b988e3d7 is 50, key is test_row_0/A:col10/1731456810042/Put/seqid=0 2024-11-13T00:13:30,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741840_1016 (size=12001) 2024-11-13T00:13:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:30,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456870836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456870838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456870847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456870847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456870848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456870957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456870958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T00:13:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456870968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456870968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456870968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,034 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/dc33743e837a411b811235a6b988e3d7 2024-11-13T00:13:31,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/6705c516a53a457c84bab76ca8b5911e is 50, key is test_row_0/B:col10/1731456810042/Put/seqid=0 2024-11-13T00:13:31,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741841_1017 (size=12001) 2024-11-13T00:13:31,078 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/6705c516a53a457c84bab76ca8b5911e 2024-11-13T00:13:31,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/06973b619b3f44e5bcad8cfb5a70f496 is 50, key is test_row_0/C:col10/1731456810042/Put/seqid=0 2024-11-13T00:13:31,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741842_1018 (size=12001) 2024-11-13T00:13:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456871168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456871168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456871189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456871189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456871189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456871478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456871478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456871497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456871507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456871510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:31,523 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/06973b619b3f44e5bcad8cfb5a70f496 2024-11-13T00:13:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/dc33743e837a411b811235a6b988e3d7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/dc33743e837a411b811235a6b988e3d7 2024-11-13T00:13:31,558 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/dc33743e837a411b811235a6b988e3d7, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T00:13:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/6705c516a53a457c84bab76ca8b5911e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6705c516a53a457c84bab76ca8b5911e 2024-11-13T00:13:31,592 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6705c516a53a457c84bab76ca8b5911e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T00:13:31,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/06973b619b3f44e5bcad8cfb5a70f496 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/06973b619b3f44e5bcad8cfb5a70f496 2024-11-13T00:13:31,614 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/06973b619b3f44e5bcad8cfb5a70f496, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T00:13:31,616 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 1047ms, sequenceid=33, compaction requested=false 2024-11-13T00:13:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T00:13:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T00:13:31,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T00:13:31,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-11-13T00:13:31,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8080 sec 2024-11-13T00:13:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:31,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:13:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T00:13:31,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:32,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T00:13:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:32,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:32,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:32,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:32,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d0738c4fd7f649d4a3bc80d88ed4198d is 50, key is test_row_0/A:col10/1731456811992/Put/seqid=0 2024-11-13T00:13:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456872043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456872048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456872052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456872055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456872057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741843_1019 (size=16681) 2024-11-13T00:13:32,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d0738c4fd7f649d4a3bc80d88ed4198d 2024-11-13T00:13:32,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/ed941b1aa8e14ee39b234baac656961f is 50, key is test_row_0/B:col10/1731456811992/Put/seqid=0 2024-11-13T00:13:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:32,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741844_1020 (size=12001) 2024-11-13T00:13:32,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456872166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456872167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456872176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456872176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456872177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456872376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456872376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456872388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456872388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456872388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/ed941b1aa8e14ee39b234baac656961f 2024-11-13T00:13:32,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/db4a7f354e8343cda2872ed26f62fb99 is 50, key is test_row_0/C:col10/1731456811992/Put/seqid=0 2024-11-13T00:13:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741845_1021 (size=12001) 2024-11-13T00:13:32,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/db4a7f354e8343cda2872ed26f62fb99 2024-11-13T00:13:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:32,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d0738c4fd7f649d4a3bc80d88ed4198d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d0738c4fd7f649d4a3bc80d88ed4198d 2024-11-13T00:13:32,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d0738c4fd7f649d4a3bc80d88ed4198d, entries=250, sequenceid=47, filesize=16.3 K 2024-11-13T00:13:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/ed941b1aa8e14ee39b234baac656961f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/ed941b1aa8e14ee39b234baac656961f 2024-11-13T00:13:32,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/ed941b1aa8e14ee39b234baac656961f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T00:13:32,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/db4a7f354e8343cda2872ed26f62fb99 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/db4a7f354e8343cda2872ed26f62fb99 2024-11-13T00:13:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456872697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456872700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/db4a7f354e8343cda2872ed26f62fb99, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T00:13:32,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 883bdd7db86faeb1b8a9722c1fd955c1 in 717ms, sequenceid=47, compaction requested=true 2024-11-13T00:13:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:32,730 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:32,730 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:32,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:13:32,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:32,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:32,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:32,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:32,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:32,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:32,739 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:32,741 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:32,742 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,742 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9fa32bfaccac47438ae2ff3a44d2a1a4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6705c516a53a457c84bab76ca8b5911e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/ed941b1aa8e14ee39b234baac656961f] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=32.9 K 2024-11-13T00:13:32,743 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:32,743 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:32,743 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,744 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa32bfaccac47438ae2ff3a44d2a1a4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731456809904 2024-11-13T00:13:32,744 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f77190f58459493dbfd8a75f25f84526, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/dc33743e837a411b811235a6b988e3d7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d0738c4fd7f649d4a3bc80d88ed4198d] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.4 K 2024-11-13T00:13:32,745 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6705c516a53a457c84bab76ca8b5911e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731456810020 2024-11-13T00:13:32,746 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ed941b1aa8e14ee39b234baac656961f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731456810833 2024-11-13T00:13:32,747 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting f77190f58459493dbfd8a75f25f84526, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731456809904 2024-11-13T00:13:32,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3f6d796699164392bb64156b1044ad97 is 50, key is test_row_0/A:col10/1731456812725/Put/seqid=0 2024-11-13T00:13:32,749 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc33743e837a411b811235a6b988e3d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731456810020 2024-11-13T00:13:32,750 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0738c4fd7f649d4a3bc80d88ed4198d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731456810824 2024-11-13T00:13:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741846_1022 (size=14341) 2024-11-13T00:13:32,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3f6d796699164392bb64156b1044ad97 2024-11-13T00:13:32,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:32,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,795 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#11 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:32,796 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#10 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,797 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/b83c1d665145477ea540efbe03beaa4d is 50, key is test_row_0/B:col10/1731456811992/Put/seqid=0 2024-11-13T00:13:32,798 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f2608bff03334edc9a8eadd3d6880e41 is 50, key is test_row_0/A:col10/1731456811992/Put/seqid=0 2024-11-13T00:13:32,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456872775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/7e23cd94512b4f6ea7e0af99d1a9b864 is 50, key is test_row_0/B:col10/1731456812725/Put/seqid=0 2024-11-13T00:13:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456872787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456872790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741847_1023 (size=12104) 2024-11-13T00:13:32,866 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/b83c1d665145477ea540efbe03beaa4d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b83c1d665145477ea540efbe03beaa4d 2024-11-13T00:13:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741848_1024 (size=12104) 2024-11-13T00:13:32,897 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into b83c1d665145477ea540efbe03beaa4d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:32,897 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:32,900 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456812729; duration=0sec 2024-11-13T00:13:32,900 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:32,901 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:32,906 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:32,912 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:32,912 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:32,912 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,912 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/1e821e16141e410ab2e566212519ec7d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/06973b619b3f44e5bcad8cfb5a70f496, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/db4a7f354e8343cda2872ed26f62fb99] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=32.9 K 2024-11-13T00:13:32,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456872908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,913 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e821e16141e410ab2e566212519ec7d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731456809904 2024-11-13T00:13:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741849_1025 (size=12001) 2024-11-13T00:13:32,917 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 06973b619b3f44e5bcad8cfb5a70f496, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731456810020 2024-11-13T00:13:32,919 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting db4a7f354e8343cda2872ed26f62fb99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731456810833 2024-11-13T00:13:32,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456872917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456872917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:32,942 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T00:13:32,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:32,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:32,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:32,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:32,971 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:32,972 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/d1e11dc465d14288a9d95e9423b73f1a is 50, key is test_row_0/C:col10/1731456811992/Put/seqid=0 2024-11-13T00:13:33,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741850_1026 (size=12104) 2024-11-13T00:13:33,023 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/d1e11dc465d14288a9d95e9423b73f1a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d1e11dc465d14288a9d95e9423b73f1a 2024-11-13T00:13:33,042 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into d1e11dc465d14288a9d95e9423b73f1a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:33,042 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:33,042 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456812729; duration=0sec 2024-11-13T00:13:33,042 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:33,043 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:33,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:33,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:33,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456873127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456873136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456873136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456873206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456873216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:33,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:33,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/7e23cd94512b4f6ea7e0af99d1a9b864 2024-11-13T00:13:33,327 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f2608bff03334edc9a8eadd3d6880e41 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2608bff03334edc9a8eadd3d6880e41 2024-11-13T00:13:33,345 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into f2608bff03334edc9a8eadd3d6880e41(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:33,345 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:33,345 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456812715; duration=0sec 2024-11-13T00:13:33,346 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:33,346 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:33,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/a2cfb863dde643cd8f898aff6b7acc19 is 50, key is test_row_0/C:col10/1731456812725/Put/seqid=0 2024-11-13T00:13:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741851_1027 (size=12001) 2024-11-13T00:13:33,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/a2cfb863dde643cd8f898aff6b7acc19 2024-11-13T00:13:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3f6d796699164392bb64156b1044ad97 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3f6d796699164392bb64156b1044ad97 2024-11-13T00:13:33,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3f6d796699164392bb64156b1044ad97, entries=200, sequenceid=67, filesize=14.0 K 2024-11-13T00:13:33,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/7e23cd94512b4f6ea7e0af99d1a9b864 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/7e23cd94512b4f6ea7e0af99d1a9b864 2024-11-13T00:13:33,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/7e23cd94512b4f6ea7e0af99d1a9b864, entries=150, sequenceid=67, filesize=11.7 K 2024-11-13T00:13:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/a2cfb863dde643cd8f898aff6b7acc19 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/a2cfb863dde643cd8f898aff6b7acc19 2024-11-13T00:13:33,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/a2cfb863dde643cd8f898aff6b7acc19, entries=150, sequenceid=67, filesize=11.7 K 2024-11-13T00:13:33,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:33,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 740ms, sequenceid=67, compaction requested=false 2024-11-13T00:13:33,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:33,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:13:33,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:33,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:33,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ac39ed145ba543cbbbec766b9f6b2d53 is 50, key is test_row_0/A:col10/1731456812743/Put/seqid=0 2024-11-13T00:13:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456873521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456873524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456873536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741852_1028 (size=12001) 2024-11-13T00:13:33,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ac39ed145ba543cbbbec766b9f6b2d53 2024-11-13T00:13:33,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1b7ea6e9d5934bb68b69b0cce1616dfa is 50, key is test_row_0/B:col10/1731456812743/Put/seqid=0 2024-11-13T00:13:33,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741853_1029 (size=12001) 2024-11-13T00:13:33,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1b7ea6e9d5934bb68b69b0cce1616dfa 2024-11-13T00:13:33,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/032470e9cca54154a2ecb3841d447ae7 is 50, key is test_row_0/C:col10/1731456812743/Put/seqid=0 2024-11-13T00:13:33,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456873646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456873647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456873647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,662 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:13:33,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741854_1030 (size=12001) 2024-11-13T00:13:33,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/032470e9cca54154a2ecb3841d447ae7 2024-11-13T00:13:33,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ac39ed145ba543cbbbec766b9f6b2d53 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac39ed145ba543cbbbec766b9f6b2d53 2024-11-13T00:13:33,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac39ed145ba543cbbbec766b9f6b2d53, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T00:13:33,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1b7ea6e9d5934bb68b69b0cce1616dfa as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1b7ea6e9d5934bb68b69b0cce1616dfa 2024-11-13T00:13:33,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:33,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:33,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1b7ea6e9d5934bb68b69b0cce1616dfa, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T00:13:33,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/032470e9cca54154a2ecb3841d447ae7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/032470e9cca54154a2ecb3841d447ae7 2024-11-13T00:13:33,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/032470e9cca54154a2ecb3841d447ae7, entries=150, sequenceid=85, filesize=11.7 K 2024-11-13T00:13:33,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 883bdd7db86faeb1b8a9722c1fd955c1 in 391ms, sequenceid=85, compaction requested=true 2024-11-13T00:13:33,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:33,863 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:33,863 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:33,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:33,866 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:33,866 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:33,866 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:33,866 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:33,868 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,868 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2608bff03334edc9a8eadd3d6880e41, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3f6d796699164392bb64156b1044ad97, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac39ed145ba543cbbbec766b9f6b2d53] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.5 K 2024-11-13T00:13:33,868 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,869 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b83c1d665145477ea540efbe03beaa4d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/7e23cd94512b4f6ea7e0af99d1a9b864, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1b7ea6e9d5934bb68b69b0cce1616dfa] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=35.3 K 2024-11-13T00:13:33,869 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2608bff03334edc9a8eadd3d6880e41, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731456810833 2024-11-13T00:13:33,870 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b83c1d665145477ea540efbe03beaa4d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731456810833 2024-11-13T00:13:33,870 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6d796699164392bb64156b1044ad97, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731456812030 2024-11-13T00:13:33,871 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e23cd94512b4f6ea7e0af99d1a9b864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731456812030 2024-11-13T00:13:33,871 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac39ed145ba543cbbbec766b9f6b2d53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731456812743 2024-11-13T00:13:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:33,873 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b7ea6e9d5934bb68b69b0cce1616dfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731456812743 2024-11-13T00:13:33,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:13:33,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:33,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:33,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:33,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:33,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:33,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:33,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f2bf1467d9634685a3a517b3b42b8568 is 50, key is test_row_0/A:col10/1731456813865/Put/seqid=0 2024-11-13T00:13:33,904 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#20 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:33,910 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/e376a83328ea4cbd8fce9bf1fea3cac8 is 50, key is test_row_0/B:col10/1731456812743/Put/seqid=0 2024-11-13T00:13:33,915 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#19 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:33,916 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d1040e8f5e4b444f99a300ea77d80bc2 is 50, key is test_row_0/A:col10/1731456812743/Put/seqid=0 2024-11-13T00:13:33,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456873930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:33,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:33,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:33,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456873936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456873936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:33,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741856_1032 (size=12207) 2024-11-13T00:13:33,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741855_1031 (size=16681) 2024-11-13T00:13:33,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f2bf1467d9634685a3a517b3b42b8568 2024-11-13T00:13:33,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741857_1033 (size=12207) 2024-11-13T00:13:33,985 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/e376a83328ea4cbd8fce9bf1fea3cac8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e376a83328ea4cbd8fce9bf1fea3cac8 2024-11-13T00:13:33,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/0ea6369e72ed49a59e98e99874a4651e is 50, key is test_row_0/B:col10/1731456813865/Put/seqid=0 2024-11-13T00:13:33,998 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d1040e8f5e4b444f99a300ea77d80bc2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1040e8f5e4b444f99a300ea77d80bc2 2024-11-13T00:13:34,005 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into e376a83328ea4cbd8fce9bf1fea3cac8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:34,005 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:34,006 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456813863; duration=0sec 2024-11-13T00:13:34,006 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:34,006 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:34,006 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:34,012 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:34,012 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:34,012 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,013 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d1e11dc465d14288a9d95e9423b73f1a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/a2cfb863dde643cd8f898aff6b7acc19, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/032470e9cca54154a2ecb3841d447ae7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=35.3 K 2024-11-13T00:13:34,014 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e11dc465d14288a9d95e9423b73f1a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731456810833 2024-11-13T00:13:34,015 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cfb863dde643cd8f898aff6b7acc19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731456812030 2024-11-13T00:13:34,017 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 032470e9cca54154a2ecb3841d447ae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731456812743 2024-11-13T00:13:34,019 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into d1040e8f5e4b444f99a300ea77d80bc2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:34,019 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:34,019 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456813863; duration=0sec 2024-11-13T00:13:34,020 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:34,020 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:34,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741858_1034 (size=12001) 2024-11-13T00:13:34,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/0ea6369e72ed49a59e98e99874a4651e 2024-11-13T00:13:34,039 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:34,040 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/41f08dfbc431452291e2f37a9089085f is 50, key is test_row_0/C:col10/1731456812743/Put/seqid=0 2024-11-13T00:13:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456874046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/872babdd735e47769cdfa3d35a684c4c is 50, key is test_row_0/C:col10/1731456813865/Put/seqid=0 2024-11-13T00:13:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456874056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456874057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741859_1035 (size=12207) 2024-11-13T00:13:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741860_1036 (size=12001) 2024-11-13T00:13:34,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:34,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:34,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,136 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/41f08dfbc431452291e2f37a9089085f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/41f08dfbc431452291e2f37a9089085f 2024-11-13T00:13:34,152 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 41f08dfbc431452291e2f37a9089085f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:34,153 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:34,153 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456813863; duration=0sec 2024-11-13T00:13:34,156 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:34,157 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:34,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:34,222 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T00:13:34,222 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T00:13:34,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T00:13:34,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T00:13:34,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T00:13:34,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T00:13:34,227 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T00:13:34,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T00:13:34,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456874226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,235 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456874236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,239 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456874256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456874267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456874276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:34,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/872babdd735e47769cdfa3d35a684c4c 2024-11-13T00:13:34,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/f2bf1467d9634685a3a517b3b42b8568 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2bf1467d9634685a3a517b3b42b8568 2024-11-13T00:13:34,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2bf1467d9634685a3a517b3b42b8568, entries=250, sequenceid=105, filesize=16.3 K 2024-11-13T00:13:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/0ea6369e72ed49a59e98e99874a4651e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0ea6369e72ed49a59e98e99874a4651e 2024-11-13T00:13:34,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0ea6369e72ed49a59e98e99874a4651e, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T00:13:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/872babdd735e47769cdfa3d35a684c4c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/872babdd735e47769cdfa3d35a684c4c 2024-11-13T00:13:34,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/872babdd735e47769cdfa3d35a684c4c, entries=150, sequenceid=105, filesize=11.7 K 2024-11-13T00:13:34,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 697ms, sequenceid=105, compaction requested=false 2024-11-13T00:13:34,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:34,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:34,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T00:13:34,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:34,572 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:13:34,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:34,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:34,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:34,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:34,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:34,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:34,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:34,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/fd6f83ca688c420589c2ef7e0afc212c is 50, key is test_row_0/A:col10/1731456814569/Put/seqid=0 2024-11-13T00:13:34,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741861_1037 (size=12001) 2024-11-13T00:13:34,607 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/fd6f83ca688c420589c2ef7e0afc212c 2024-11-13T00:13:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/fb8fde2edd944bf39e2f249bf48987d0 is 50, key is test_row_0/B:col10/1731456814569/Put/seqid=0 2024-11-13T00:13:34,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741862_1038 (size=12001) 2024-11-13T00:13:34,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456874631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456874635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456874636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456874746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456874746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456874747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456874956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456874957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:34,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456874966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,034 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/fb8fde2edd944bf39e2f249bf48987d0 2024-11-13T00:13:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/d782bc47a7074628be8e2a3e3a393d69 is 50, key is test_row_0/C:col10/1731456814569/Put/seqid=0 2024-11-13T00:13:35,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741863_1039 (size=12001) 2024-11-13T00:13:35,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456875266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456875266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456875276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,462 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/d782bc47a7074628be8e2a3e3a393d69 2024-11-13T00:13:35,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/fd6f83ca688c420589c2ef7e0afc212c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fd6f83ca688c420589c2ef7e0afc212c 2024-11-13T00:13:35,481 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fd6f83ca688c420589c2ef7e0afc212c, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T00:13:35,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/fb8fde2edd944bf39e2f249bf48987d0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/fb8fde2edd944bf39e2f249bf48987d0 2024-11-13T00:13:35,491 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/fb8fde2edd944bf39e2f249bf48987d0, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T00:13:35,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/d782bc47a7074628be8e2a3e3a393d69 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d782bc47a7074628be8e2a3e3a393d69 2024-11-13T00:13:35,501 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d782bc47a7074628be8e2a3e3a393d69, entries=150, sequenceid=121, filesize=11.7 K 2024-11-13T00:13:35,503 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 883bdd7db86faeb1b8a9722c1fd955c1 in 931ms, sequenceid=121, compaction requested=true 2024-11-13T00:13:35,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:35,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:35,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T00:13:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T00:13:35,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T00:13:35,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5000 sec 2024-11-13T00:13:35,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 3.5110 sec 2024-11-13T00:13:35,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T00:13:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:35,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/59fbfe17ad024a2db2131dcf77d447d5 is 50, key is test_row_0/A:col10/1731456814628/Put/seqid=0 2024-11-13T00:13:35,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741864_1040 (size=12151) 2024-11-13T00:13:35,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456875845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456875845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456875853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456875956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456875956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456875966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456876166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456876166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T00:13:36,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456876176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T00:13:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T00:13:36,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:36,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:36,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:36,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/59fbfe17ad024a2db2131dcf77d447d5 2024-11-13T00:13:36,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/6e90f3e2ee9243acb7bcbecd728df3fb is 50, key is test_row_0/B:col10/1731456814628/Put/seqid=0 2024-11-13T00:13:36,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741865_1041 (size=12151) 2024-11-13T00:13:36,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/6e90f3e2ee9243acb7bcbecd728df3fb 2024-11-13T00:13:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456876248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,250 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/52630e3c27d845318de1abb379661b59 is 50, key is test_row_0/C:col10/1731456814628/Put/seqid=0 2024-11-13T00:13:36,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456876267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,269 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741866_1042 (size=12151) 2024-11-13T00:13:36,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/52630e3c27d845318de1abb379661b59 2024-11-13T00:13:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T00:13:36,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/59fbfe17ad024a2db2131dcf77d447d5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/59fbfe17ad024a2db2131dcf77d447d5 2024-11-13T00:13:36,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/59fbfe17ad024a2db2131dcf77d447d5, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T00:13:36,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/6e90f3e2ee9243acb7bcbecd728df3fb as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6e90f3e2ee9243acb7bcbecd728df3fb 2024-11-13T00:13:36,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6e90f3e2ee9243acb7bcbecd728df3fb, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T00:13:36,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/52630e3c27d845318de1abb379661b59 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/52630e3c27d845318de1abb379661b59 2024-11-13T00:13:36,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/52630e3c27d845318de1abb379661b59, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T00:13:36,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 883bdd7db86faeb1b8a9722c1fd955c1 in 537ms, sequenceid=142, compaction requested=true 2024-11-13T00:13:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:36,322 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:36,322 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:36,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:36,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:36,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:36,325 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:36,325 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:36,325 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:36,325 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e376a83328ea4cbd8fce9bf1fea3cac8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0ea6369e72ed49a59e98e99874a4651e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/fb8fde2edd944bf39e2f249bf48987d0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6e90f3e2ee9243acb7bcbecd728df3fb] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=47.2 K 2024-11-13T00:13:36,327 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53040 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:36,327 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:36,327 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:36,327 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1040e8f5e4b444f99a300ea77d80bc2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2bf1467d9634685a3a517b3b42b8568, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fd6f83ca688c420589c2ef7e0afc212c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/59fbfe17ad024a2db2131dcf77d447d5] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=51.8 K 2024-11-13T00:13:36,328 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e376a83328ea4cbd8fce9bf1fea3cac8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731456812743 2024-11-13T00:13:36,328 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1040e8f5e4b444f99a300ea77d80bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731456812743 2024-11-13T00:13:36,328 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea6369e72ed49a59e98e99874a4651e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731456813520 2024-11-13T00:13:36,329 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8fde2edd944bf39e2f249bf48987d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731456813927 2024-11-13T00:13:36,329 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2bf1467d9634685a3a517b3b42b8568, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731456813517 2024-11-13T00:13:36,330 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e90f3e2ee9243acb7bcbecd728df3fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456814628 2024-11-13T00:13:36,330 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6f83ca688c420589c2ef7e0afc212c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731456813927 2024-11-13T00:13:36,332 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59fbfe17ad024a2db2131dcf77d447d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456814628 2024-11-13T00:13:36,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T00:13:36,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:36,338 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:13:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/20305fcb7a8d41d188944fb43bf7dbad is 50, key is test_row_0/A:col10/1731456815831/Put/seqid=0 2024-11-13T00:13:36,355 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:36,356 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/0cfcdb6503194555919768b059d395da is 50, key is test_row_0/B:col10/1731456814628/Put/seqid=0 2024-11-13T00:13:36,360 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#32 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:36,361 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/9fe7924361a24cddbd91c8ea6667c597 is 50, key is test_row_0/A:col10/1731456814628/Put/seqid=0 2024-11-13T00:13:36,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741867_1043 (size=12151) 2024-11-13T00:13:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741868_1044 (size=12493) 2024-11-13T00:13:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741869_1045 (size=12493) 2024-11-13T00:13:36,407 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/0cfcdb6503194555919768b059d395da as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0cfcdb6503194555919768b059d395da 2024-11-13T00:13:36,420 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into 0cfcdb6503194555919768b059d395da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:36,420 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:36,420 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=12, startTime=1731456816322; duration=0sec 2024-11-13T00:13:36,421 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:36,421 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:36,421 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:36,423 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:36,423 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:36,423 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:36,423 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/41f08dfbc431452291e2f37a9089085f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/872babdd735e47769cdfa3d35a684c4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d782bc47a7074628be8e2a3e3a393d69, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/52630e3c27d845318de1abb379661b59] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=47.2 K 2024-11-13T00:13:36,424 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f08dfbc431452291e2f37a9089085f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731456812743 2024-11-13T00:13:36,425 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 872babdd735e47769cdfa3d35a684c4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731456813520 2024-11-13T00:13:36,426 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d782bc47a7074628be8e2a3e3a393d69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731456813927 2024-11-13T00:13:36,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 52630e3c27d845318de1abb379661b59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456814628 2024-11-13T00:13:36,448 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:36,449 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/503aba9d09be4d3f861bb247ca73ffd5 is 50, key is test_row_0/C:col10/1731456814628/Put/seqid=0 2024-11-13T00:13:36,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741870_1046 (size=12493) 2024-11-13T00:13:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:36,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T00:13:36,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456816509, max timestamp=1731456816526Num uniques 859; 2024-11-13T00:13:36,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:13:36,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731456816509, max timestamp=1731456816526Num uniques 859; 2024-11-13T00:13:36,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T00:13:36,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731456816509, max timestamp=1731456816526Num uniques 859; 2024-11-13T00:13:36,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T00:13:36,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456876600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456876601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456876601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456876716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456876716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456876726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:36,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:36,776 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/20305fcb7a8d41d188944fb43bf7dbad 2024-11-13T00:13:36,787 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/9fe7924361a24cddbd91c8ea6667c597 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/9fe7924361a24cddbd91c8ea6667c597 2024-11-13T00:13:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/739c921be5d24585a2828e1c0c99a0cc is 50, key is test_row_0/B:col10/1731456815831/Put/seqid=0 2024-11-13T00:13:36,801 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 9fe7924361a24cddbd91c8ea6667c597(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:36,801 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:36,801 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=12, startTime=1731456816322; duration=0sec 2024-11-13T00:13:36,802 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:36,802 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T00:13:36,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741871_1047 (size=12151) 2024-11-13T00:13:36,812 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/739c921be5d24585a2828e1c0c99a0cc 2024-11-13T00:13:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/08aef294a9fa4621b351f2218bcebc95 is 50, key is test_row_0/C:col10/1731456815831/Put/seqid=0 2024-11-13T00:13:36,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741872_1048 (size=12151) 2024-11-13T00:13:36,848 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/08aef294a9fa4621b351f2218bcebc95 2024-11-13T00:13:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/20305fcb7a8d41d188944fb43bf7dbad as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/20305fcb7a8d41d188944fb43bf7dbad 2024-11-13T00:13:36,872 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/20305fcb7a8d41d188944fb43bf7dbad, entries=150, sequenceid=153, filesize=11.9 K 2024-11-13T00:13:36,872 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/503aba9d09be4d3f861bb247ca73ffd5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/503aba9d09be4d3f861bb247ca73ffd5 2024-11-13T00:13:36,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/739c921be5d24585a2828e1c0c99a0cc as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/739c921be5d24585a2828e1c0c99a0cc 2024-11-13T00:13:36,883 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 503aba9d09be4d3f861bb247ca73ffd5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:36,884 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:36,884 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=12, startTime=1731456816323; duration=0sec 2024-11-13T00:13:36,884 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:36,886 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:36,885 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/739c921be5d24585a2828e1c0c99a0cc, entries=150, sequenceid=153, filesize=11.9 K 2024-11-13T00:13:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/08aef294a9fa4621b351f2218bcebc95 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/08aef294a9fa4621b351f2218bcebc95 2024-11-13T00:13:36,893 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/08aef294a9fa4621b351f2218bcebc95, entries=150, sequenceid=153, filesize=11.9 K 2024-11-13T00:13:36,895 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.39 KB/136590 for 883bdd7db86faeb1b8a9722c1fd955c1 in 557ms, sequenceid=153, compaction requested=false 2024-11-13T00:13:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T00:13:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T00:13:36,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T00:13:36,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 714 msec 2024-11-13T00:13:36,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 724 msec 2024-11-13T00:13:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:36,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=140.10 KB heapSize=403.78 KB 2024-11-13T00:13:36,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456816596, max timestamp=1731456816939Num uniques 1018; 2024-11-13T00:13:36,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:13:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456816596, max timestamp=1731456816939Num uniques 1009; 2024-11-13T00:13:36,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:36,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-13T00:13:36,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:36,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:36,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:36,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:36,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:13:36,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/80f7b076d90a45b9b14026678e55f9c7 is 50, key is test_row_0/A:col10/1731456816941/Put/seqid=0 2024-11-13T00:13:36,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741873_1049 (size=12151) 2024-11-13T00:13:36,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/80f7b076d90a45b9b14026678e55f9c7 2024-11-13T00:13:36,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/f964af3e8373415ab06e6df456bb6fc9 is 50, key is test_row_0/B:col10/1731456816941/Put/seqid=0 2024-11-13T00:13:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456876997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456877001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456877003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741874_1050 (size=12151) 2024-11-13T00:13:37,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/f964af3e8373415ab06e6df456bb6fc9 2024-11-13T00:13:37,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/e0d8e56ce5984aaaac13f2f1475eb94a is 50, key is test_row_0/C:col10/1731456816941/Put/seqid=0 2024-11-13T00:13:37,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741875_1051 (size=12151) 2024-11-13T00:13:37,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.04 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/e0d8e56ce5984aaaac13f2f1475eb94a 2024-11-13T00:13:37,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/80f7b076d90a45b9b14026678e55f9c7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/80f7b076d90a45b9b14026678e55f9c7 2024-11-13T00:13:37,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/80f7b076d90a45b9b14026678e55f9c7, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T00:13:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/f964af3e8373415ab06e6df456bb6fc9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/f964af3e8373415ab06e6df456bb6fc9 2024-11-13T00:13:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456877117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456877117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456877118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/f964af3e8373415ab06e6df456bb6fc9, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T00:13:37,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/e0d8e56ce5984aaaac13f2f1475eb94a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/e0d8e56ce5984aaaac13f2f1475eb94a 2024-11-13T00:13:37,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/e0d8e56ce5984aaaac13f2f1475eb94a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T00:13:37,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~77.88 KB/79745, heapSize ~217.68 KB/222904, currentSize=100.63 KB/103050 for 883bdd7db86faeb1b8a9722c1fd955c1 in 209ms, sequenceid=196, compaction requested=true 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:37,150 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:37,150 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:37,153 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:37,153 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:37,153 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:37,153 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0cfcdb6503194555919768b059d395da, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/739c921be5d24585a2828e1c0c99a0cc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/f964af3e8373415ab06e6df456bb6fc9] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=35.9 K 2024-11-13T00:13:37,154 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:37,154 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:37,154 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:37,154 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/9fe7924361a24cddbd91c8ea6667c597, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/20305fcb7a8d41d188944fb43bf7dbad, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/80f7b076d90a45b9b14026678e55f9c7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=35.9 K 2024-11-13T00:13:37,154 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cfcdb6503194555919768b059d395da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456814628 2024-11-13T00:13:37,154 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe7924361a24cddbd91c8ea6667c597, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456814628 2024-11-13T00:13:37,155 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 739c921be5d24585a2828e1c0c99a0cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456815797 2024-11-13T00:13:37,155 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20305fcb7a8d41d188944fb43bf7dbad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456815797 2024-11-13T00:13:37,155 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f964af3e8373415ab06e6df456bb6fc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731456816939 2024-11-13T00:13:37,155 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f7b076d90a45b9b14026678e55f9c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731456816939 2024-11-13T00:13:37,182 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:37,182 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:37,183 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/a95a7a6ed8e04df7ad97db3305137e56 is 50, key is test_row_0/B:col10/1731456816941/Put/seqid=0 2024-11-13T00:13:37,183 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/2f912d600a7a4ab59c95138882195172 is 50, key is test_row_0/A:col10/1731456816941/Put/seqid=0 2024-11-13T00:13:37,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741876_1052 (size=12595) 2024-11-13T00:13:37,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741877_1053 (size=12595) 2024-11-13T00:13:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T00:13:37,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:37,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T00:13:37,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T00:13:37,322 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:37,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:37,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:13:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:37,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/70f672d2170741f995530554e4cc0df5 is 50, key is test_row_0/A:col10/1731456816997/Put/seqid=0 2024-11-13T00:13:37,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741878_1054 (size=12151) 2024-11-13T00:13:37,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/70f672d2170741f995530554e4cc0df5 2024-11-13T00:13:37,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/494934b6ad084302b5e87d5fc75d6197 is 50, key is test_row_0/B:col10/1731456816997/Put/seqid=0 2024-11-13T00:13:37,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741879_1055 (size=12151) 2024-11-13T00:13:37,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/494934b6ad084302b5e87d5fc75d6197 2024-11-13T00:13:37,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/aea029f0ccaa40a797f0e875ff2175b5 is 50, key is test_row_0/C:col10/1731456816997/Put/seqid=0 2024-11-13T00:13:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741880_1056 (size=12151) 2024-11-13T00:13:37,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/aea029f0ccaa40a797f0e875ff2175b5 2024-11-13T00:13:37,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/70f672d2170741f995530554e4cc0df5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/70f672d2170741f995530554e4cc0df5 2024-11-13T00:13:37,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456877403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/70f672d2170741f995530554e4cc0df5, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T00:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456877403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/494934b6ad084302b5e87d5fc75d6197 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/494934b6ad084302b5e87d5fc75d6197 2024-11-13T00:13:37,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456877411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/494934b6ad084302b5e87d5fc75d6197, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T00:13:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/aea029f0ccaa40a797f0e875ff2175b5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/aea029f0ccaa40a797f0e875ff2175b5 2024-11-13T00:13:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T00:13:37,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/aea029f0ccaa40a797f0e875ff2175b5, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T00:13:37,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 101ms, sequenceid=216, compaction requested=true 2024-11-13T00:13:37,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T00:13:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T00:13:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T00:13:37,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T00:13:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:37,481 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:13:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/afdca85527624769ac73369ebbaf4db1 is 50, key is test_row_0/A:col10/1731456817348/Put/seqid=0 2024-11-13T00:13:37,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741881_1057 (size=9757) 2024-11-13T00:13:37,509 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/afdca85527624769ac73369ebbaf4db1 2024-11-13T00:13:37,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:37,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d is 50, key is test_row_0/B:col10/1731456817348/Put/seqid=0 2024-11-13T00:13:37,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741882_1058 (size=9757) 2024-11-13T00:13:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456877592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456877602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456877602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,617 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/a95a7a6ed8e04df7ad97db3305137e56 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/a95a7a6ed8e04df7ad97db3305137e56 2024-11-13T00:13:37,621 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/2f912d600a7a4ab59c95138882195172 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2f912d600a7a4ab59c95138882195172 2024-11-13T00:13:37,636 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into a95a7a6ed8e04df7ad97db3305137e56(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:37,636 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:37,636 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456817150; duration=0sec 2024-11-13T00:13:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T00:13:37,637 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T00:13:37,637 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:37,637 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:37,637 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:37,637 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 2f912d600a7a4ab59c95138882195172(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:37,637 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:37,637 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456817150; duration=0sec 2024-11-13T00:13:37,638 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T00:13:37,638 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:37,638 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:37,639 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:37,639 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:37,639 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-13T00:13:37,639 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:37,639 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T00:13:37,640 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T00:13:37,640 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/503aba9d09be4d3f861bb247ca73ffd5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/08aef294a9fa4621b351f2218bcebc95, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/e0d8e56ce5984aaaac13f2f1475eb94a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/aea029f0ccaa40a797f0e875ff2175b5] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=47.8 K 2024-11-13T00:13:37,641 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. because compaction request was cancelled 2024-11-13T00:13:37,641 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:37,641 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 503aba9d09be4d3f861bb247ca73ffd5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456814628 2024-11-13T00:13:37,641 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T00:13:37,642 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 08aef294a9fa4621b351f2218bcebc95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456815797 2024-11-13T00:13:37,642 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T00:13:37,642 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d8e56ce5984aaaac13f2f1475eb94a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731456816939 2024-11-13T00:13:37,643 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T00:13:37,643 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. because compaction request was cancelled 2024-11-13T00:13:37,643 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:37,643 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting aea029f0ccaa40a797f0e875ff2175b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456816997 2024-11-13T00:13:37,643 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T00:13:37,644 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T00:13:37,644 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T00:13:37,645 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. because compaction request was cancelled 2024-11-13T00:13:37,645 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:37,660 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:37,661 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/83ea5cff08524971b771372d97a784be is 50, key is test_row_0/C:col10/1731456816997/Put/seqid=0 2024-11-13T00:13:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741883_1059 (size=12629) 2024-11-13T00:13:37,684 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/83ea5cff08524971b771372d97a784be as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/83ea5cff08524971b771372d97a784be 2024-11-13T00:13:37,697 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 83ea5cff08524971b771372d97a784be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:37,697 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:37,697 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=12, startTime=1731456817435; duration=0sec 2024-11-13T00:13:37,698 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:37,698 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456877707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456877707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456877716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456877916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456877917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456877926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T00:13:37,958 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d 2024-11-13T00:13:37,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/34205403a26e4fd891f2d244582f1822 is 50, key is test_row_0/C:col10/1731456817348/Put/seqid=0 2024-11-13T00:13:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741884_1060 (size=9757) 2024-11-13T00:13:38,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456878226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456878227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456878236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,386 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/34205403a26e4fd891f2d244582f1822 2024-11-13T00:13:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/afdca85527624769ac73369ebbaf4db1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/afdca85527624769ac73369ebbaf4db1 2024-11-13T00:13:38,399 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/afdca85527624769ac73369ebbaf4db1, entries=100, sequenceid=228, filesize=9.5 K 2024-11-13T00:13:38,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d 2024-11-13T00:13:38,406 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d, entries=100, sequenceid=228, filesize=9.5 K 2024-11-13T00:13:38,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/34205403a26e4fd891f2d244582f1822 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34205403a26e4fd891f2d244582f1822 2024-11-13T00:13:38,413 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34205403a26e4fd891f2d244582f1822, entries=100, sequenceid=228, filesize=9.5 K 2024-11-13T00:13:38,414 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 883bdd7db86faeb1b8a9722c1fd955c1 in 933ms, sequenceid=228, compaction requested=true 2024-11-13T00:13:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T00:13:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T00:13:38,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T00:13:38,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0950 sec 2024-11-13T00:13:38,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1040 sec 2024-11-13T00:13:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T00:13:38,456 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:38,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T00:13:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T00:13:38,463 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:38,464 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:38,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T00:13:38,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T00:13:38,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:38,617 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:13:38,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:38,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:38,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:38,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:38,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:38,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:38,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/51f57b29ed5847ca9d5477401846b605 is 50, key is test_row_0/A:col10/1731456817599/Put/seqid=0 2024-11-13T00:13:38,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741885_1061 (size=12151) 2024-11-13T00:13:38,657 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/51f57b29ed5847ca9d5477401846b605 2024-11-13T00:13:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/c58753577fb14412b8e767e51bdea617 is 50, key is test_row_0/B:col10/1731456817599/Put/seqid=0 2024-11-13T00:13:38,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741886_1062 (size=12151) 2024-11-13T00:13:38,683 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/c58753577fb14412b8e767e51bdea617 2024-11-13T00:13:38,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/fd1989f118fa4da8b9e61c81af4dfd32 is 50, key is test_row_0/C:col10/1731456817599/Put/seqid=0 2024-11-13T00:13:38,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741887_1063 (size=12151) 2024-11-13T00:13:38,712 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/fd1989f118fa4da8b9e61c81af4dfd32 2024-11-13T00:13:38,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/51f57b29ed5847ca9d5477401846b605 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/51f57b29ed5847ca9d5477401846b605 2024-11-13T00:13:38,728 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/51f57b29ed5847ca9d5477401846b605, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T00:13:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/c58753577fb14412b8e767e51bdea617 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/c58753577fb14412b8e767e51bdea617 2024-11-13T00:13:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:38,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:38,740 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/c58753577fb14412b8e767e51bdea617, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T00:13:38,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/fd1989f118fa4da8b9e61c81af4dfd32 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/fd1989f118fa4da8b9e61c81af4dfd32 2024-11-13T00:13:38,751 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/fd1989f118fa4da8b9e61c81af4dfd32, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T00:13:38,755 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=53.67 KB/54960 for 883bdd7db86faeb1b8a9722c1fd955c1 in 138ms, sequenceid=251, compaction requested=true 2024-11-13T00:13:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T00:13:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T00:13:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:38,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:13:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:38,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T00:13:38,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 293 msec 2024-11-13T00:13:38,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 305 msec 2024-11-13T00:13:38,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/e6ba9921d30d4acabf39b3ef374e0a43 is 50, key is test_row_0/A:col10/1731456818741/Put/seqid=0 2024-11-13T00:13:38,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741888_1064 (size=17131) 2024-11-13T00:13:38,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/e6ba9921d30d4acabf39b3ef374e0a43 2024-11-13T00:13:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T00:13:38,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:38,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T00:13:38,782 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:38,783 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:38,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T00:13:38,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/041d536b71904ab684d0b7fdca483fce is 50, key is test_row_0/B:col10/1731456818741/Put/seqid=0 2024-11-13T00:13:38,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741889_1065 (size=12251) 2024-11-13T00:13:38,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456878823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456878827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456878828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T00:13:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456878936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456878936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T00:13:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:38,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456878936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:38,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T00:13:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T00:13:39,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456879146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456879147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456879147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/041d536b71904ab684d0b7fdca483fce 2024-11-13T00:13:39,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/ff3bedd030f84d2eb05ace3a3503d5a9 is 50, key is test_row_0/C:col10/1731456818741/Put/seqid=0 2024-11-13T00:13:39,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T00:13:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:39,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741890_1066 (size=12251) 2024-11-13T00:13:39,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/ff3bedd030f84d2eb05ace3a3503d5a9 2024-11-13T00:13:39,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/e6ba9921d30d4acabf39b3ef374e0a43 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/e6ba9921d30d4acabf39b3ef374e0a43 2024-11-13T00:13:39,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/e6ba9921d30d4acabf39b3ef374e0a43, entries=250, sequenceid=263, filesize=16.7 K 2024-11-13T00:13:39,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/041d536b71904ab684d0b7fdca483fce as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/041d536b71904ab684d0b7fdca483fce 2024-11-13T00:13:39,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/041d536b71904ab684d0b7fdca483fce, entries=150, sequenceid=263, filesize=12.0 K 2024-11-13T00:13:39,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/ff3bedd030f84d2eb05ace3a3503d5a9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/ff3bedd030f84d2eb05ace3a3503d5a9 2024-11-13T00:13:39,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/ff3bedd030f84d2eb05ace3a3503d5a9, entries=150, sequenceid=263, filesize=12.0 K 2024-11-13T00:13:39,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 883bdd7db86faeb1b8a9722c1fd955c1 in 564ms, sequenceid=263, compaction requested=true 2024-11-13T00:13:39,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:39,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:39,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:39,320 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T00:13:39,321 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T00:13:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:39,323 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58905 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T00:13:39,323 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:39,324 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,324 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/a95a7a6ed8e04df7ad97db3305137e56, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/494934b6ad084302b5e87d5fc75d6197, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/c58753577fb14412b8e767e51bdea617, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/041d536b71904ab684d0b7fdca483fce] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=57.5 K 2024-11-13T00:13:39,324 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63785 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T00:13:39,324 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:39,324 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,325 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a95a7a6ed8e04df7ad97db3305137e56, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731456816939 2024-11-13T00:13:39,325 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2f912d600a7a4ab59c95138882195172, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/70f672d2170741f995530554e4cc0df5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/afdca85527624769ac73369ebbaf4db1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/51f57b29ed5847ca9d5477401846b605, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/e6ba9921d30d4acabf39b3ef374e0a43] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=62.3 K 2024-11-13T00:13:39,325 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 494934b6ad084302b5e87d5fc75d6197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456816997 2024-11-13T00:13:39,325 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f912d600a7a4ab59c95138882195172, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731456816939 2024-11-13T00:13:39,326 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 12c3fbda8c4c40cc9bbf8deecb5c6f0d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731456817348 2024-11-13T00:13:39,326 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70f672d2170741f995530554e4cc0df5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456816997 2024-11-13T00:13:39,327 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting afdca85527624769ac73369ebbaf4db1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731456817348 2024-11-13T00:13:39,327 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting c58753577fb14412b8e767e51bdea617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731456817589 2024-11-13T00:13:39,328 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f57b29ed5847ca9d5477401846b605, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731456817589 2024-11-13T00:13:39,329 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ba9921d30d4acabf39b3ef374e0a43, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731456818741 2024-11-13T00:13:39,329 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 041d536b71904ab684d0b7fdca483fce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731456818741 2024-11-13T00:13:39,352 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:39,352 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#54 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:39,353 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/bfe7b43f58b24d4196000dc3724cef61 is 50, key is test_row_0/B:col10/1731456818741/Put/seqid=0 2024-11-13T00:13:39,353 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0246f33320a44014840058f45803c745 is 50, key is test_row_0/A:col10/1731456818741/Put/seqid=0 2024-11-13T00:13:39,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741891_1067 (size=12865) 2024-11-13T00:13:39,384 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0246f33320a44014840058f45803c745 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0246f33320a44014840058f45803c745 2024-11-13T00:13:39,391 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 0246f33320a44014840058f45803c745(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:39,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:39,391 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=11, startTime=1731456819320; duration=0sec 2024-11-13T00:13:39,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:39,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:39,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:39,393 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46788 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:39,393 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:39,393 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741892_1068 (size=12865) 2024-11-13T00:13:39,395 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/83ea5cff08524971b771372d97a784be, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34205403a26e4fd891f2d244582f1822, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/fd1989f118fa4da8b9e61c81af4dfd32, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/ff3bedd030f84d2eb05ace3a3503d5a9] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=45.7 K 2024-11-13T00:13:39,395 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ea5cff08524971b771372d97a784be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456816997 2024-11-13T00:13:39,396 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34205403a26e4fd891f2d244582f1822, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731456817348 2024-11-13T00:13:39,396 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1989f118fa4da8b9e61c81af4dfd32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731456817589 2024-11-13T00:13:39,397 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3bedd030f84d2eb05ace3a3503d5a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731456818741 2024-11-13T00:13:39,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T00:13:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,402 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:13:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/da0e95048ec24ec38f84aee69d73ac6d is 50, key is test_row_0/A:col10/1731456818804/Put/seqid=0 2024-11-13T00:13:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T00:13:39,419 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:39,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741893_1069 (size=12301) 2024-11-13T00:13:39,420 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/17838e8111b042e1b5db9c946102b613 is 50, key is test_row_0/C:col10/1731456818741/Put/seqid=0 2024-11-13T00:13:39,420 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/da0e95048ec24ec38f84aee69d73ac6d 2024-11-13T00:13:39,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741894_1070 (size=12865) 2024-11-13T00:13:39,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/e82b6f11548349f0a599ba5551073019 is 50, key is test_row_0/B:col10/1731456818804/Put/seqid=0 2024-11-13T00:13:39,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741895_1071 (size=12301) 2024-11-13T00:13:39,455 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/e82b6f11548349f0a599ba5551073019 2024-11-13T00:13:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:39,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/334fd291baa540a6aeb6cc4eeb2b8bb2 is 50, key is test_row_0/C:col10/1731456818804/Put/seqid=0 2024-11-13T00:13:39,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456879482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456879484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456879486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741896_1072 (size=12301) 2024-11-13T00:13:39,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456879596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456879596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456879596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,802 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/bfe7b43f58b24d4196000dc3724cef61 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/bfe7b43f58b24d4196000dc3724cef61 2024-11-13T00:13:39,809 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into bfe7b43f58b24d4196000dc3724cef61(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:39,809 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,809 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=11, startTime=1731456819320; duration=0sec 2024-11-13T00:13:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456879806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,809 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:39,809 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456879806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456879806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:39,834 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/17838e8111b042e1b5db9c946102b613 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/17838e8111b042e1b5db9c946102b613 2024-11-13T00:13:39,841 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 17838e8111b042e1b5db9c946102b613(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:39,841 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:39,841 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=12, startTime=1731456819321; duration=0sec 2024-11-13T00:13:39,842 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:39,842 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:39,901 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/334fd291baa540a6aeb6cc4eeb2b8bb2 2024-11-13T00:13:39,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/da0e95048ec24ec38f84aee69d73ac6d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/da0e95048ec24ec38f84aee69d73ac6d 2024-11-13T00:13:39,914 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/da0e95048ec24ec38f84aee69d73ac6d, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T00:13:39,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/e82b6f11548349f0a599ba5551073019 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e82b6f11548349f0a599ba5551073019 2024-11-13T00:13:39,921 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e82b6f11548349f0a599ba5551073019, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T00:13:39,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/334fd291baa540a6aeb6cc4eeb2b8bb2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/334fd291baa540a6aeb6cc4eeb2b8bb2 2024-11-13T00:13:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T00:13:39,930 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/334fd291baa540a6aeb6cc4eeb2b8bb2, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T00:13:39,931 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 529ms, sequenceid=284, compaction requested=false 2024-11-13T00:13:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T00:13:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T00:13:39,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T00:13:39,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1530 sec 2024-11-13T00:13:39,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1620 sec 2024-11-13T00:13:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:40,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:13:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:40,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/33ef34b3a83c42ec9224470cda0a7a23 is 50, key is test_row_0/A:col10/1731456820118/Put/seqid=0 2024-11-13T00:13:40,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741897_1073 (size=17181) 2024-11-13T00:13:40,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/33ef34b3a83c42ec9224470cda0a7a23 2024-11-13T00:13:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456880154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/073bb664c84b42a6b20edcfd66bcae18 is 50, key is test_row_0/B:col10/1731456820118/Put/seqid=0 2024-11-13T00:13:40,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456880156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456880158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741898_1074 (size=12301) 2024-11-13T00:13:40,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456880266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456880266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456880276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456880286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456880286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,288 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,288 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456880476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456880476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456880486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/073bb664c84b42a6b20edcfd66bcae18 2024-11-13T00:13:40,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/25b4767aab9b4257beb388e6ebd27f17 is 50, key is test_row_0/C:col10/1731456820118/Put/seqid=0 2024-11-13T00:13:40,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741899_1075 (size=12301) 2024-11-13T00:13:40,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/25b4767aab9b4257beb388e6ebd27f17 2024-11-13T00:13:40,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/33ef34b3a83c42ec9224470cda0a7a23 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/33ef34b3a83c42ec9224470cda0a7a23 2024-11-13T00:13:40,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/33ef34b3a83c42ec9224470cda0a7a23, entries=250, sequenceid=300, filesize=16.8 K 2024-11-13T00:13:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/073bb664c84b42a6b20edcfd66bcae18 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/073bb664c84b42a6b20edcfd66bcae18 2024-11-13T00:13:40,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/073bb664c84b42a6b20edcfd66bcae18, entries=150, sequenceid=300, filesize=12.0 K 2024-11-13T00:13:40,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/25b4767aab9b4257beb388e6ebd27f17 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/25b4767aab9b4257beb388e6ebd27f17 2024-11-13T00:13:40,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/25b4767aab9b4257beb388e6ebd27f17, entries=150, sequenceid=300, filesize=12.0 K 2024-11-13T00:13:40,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 883bdd7db86faeb1b8a9722c1fd955c1 in 517ms, sequenceid=300, compaction requested=true 2024-11-13T00:13:40,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:40,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:40,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:40,635 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:40,635 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:40,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:40,637 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:40,637 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:40,637 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:40,637 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/bfe7b43f58b24d4196000dc3724cef61, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e82b6f11548349f0a599ba5551073019, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/073bb664c84b42a6b20edcfd66bcae18] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=36.6 K 2024-11-13T00:13:40,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:40,637 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:40,638 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe7b43f58b24d4196000dc3724cef61, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731456818741 2024-11-13T00:13:40,638 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:40,638 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:40,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:40,638 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0246f33320a44014840058f45803c745, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/da0e95048ec24ec38f84aee69d73ac6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/33ef34b3a83c42ec9224470cda0a7a23] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=41.4 K 2024-11-13T00:13:40,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:40,638 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e82b6f11548349f0a599ba5551073019, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731456818804 2024-11-13T00:13:40,638 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0246f33320a44014840058f45803c745, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731456818741 2024-11-13T00:13:40,639 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 073bb664c84b42a6b20edcfd66bcae18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731456819481 2024-11-13T00:13:40,639 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0e95048ec24ec38f84aee69d73ac6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731456818804 2024-11-13T00:13:40,640 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ef34b3a83c42ec9224470cda0a7a23, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731456819472 2024-11-13T00:13:40,652 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:40,654 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/a07e938e61234566b60351de9ca1d0a2 is 50, key is test_row_0/A:col10/1731456820118/Put/seqid=0 2024-11-13T00:13:40,659 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:40,659 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0 is 50, key is test_row_0/B:col10/1731456820118/Put/seqid=0 2024-11-13T00:13:40,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741900_1076 (size=13017) 2024-11-13T00:13:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741901_1077 (size=13017) 2024-11-13T00:13:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:40,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:13:40,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:40,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:40,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:40,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:40,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:40,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:40,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/2975749efdd34bbd9d9d65dc95b3b11f is 50, key is test_row_0/A:col10/1731456820150/Put/seqid=0 2024-11-13T00:13:40,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741902_1078 (size=14741) 2024-11-13T00:13:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456880809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456880811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456880812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456880916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456880926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456880927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T00:13:40,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:40,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-13T00:13:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T00:13:40,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:40,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:40,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T00:13:41,080 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/a07e938e61234566b60351de9ca1d0a2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/a07e938e61234566b60351de9ca1d0a2 2024-11-13T00:13:41,083 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0 2024-11-13T00:13:41,093 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into a07e938e61234566b60351de9ca1d0a2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:41,093 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:41,093 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456820635; duration=0sec 2024-11-13T00:13:41,093 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:41,093 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:41,093 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:41,094 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into db5cdeb4c3d84ec48ba9265d8e9ac5d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:41,094 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:41,094 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456820635; duration=0sec 2024-11-13T00:13:41,094 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:41,095 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:41,096 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:41,097 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:41,097 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,097 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/17838e8111b042e1b5db9c946102b613, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/334fd291baa540a6aeb6cc4eeb2b8bb2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/25b4767aab9b4257beb388e6ebd27f17] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=36.6 K 2024-11-13T00:13:41,097 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17838e8111b042e1b5db9c946102b613, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731456818741 2024-11-13T00:13:41,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T00:13:41,098 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 334fd291baa540a6aeb6cc4eeb2b8bb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731456818804 2024-11-13T00:13:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,099 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25b4767aab9b4257beb388e6ebd27f17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731456819481 2024-11-13T00:13:41,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,111 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:41,111 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d is 50, key is test_row_0/C:col10/1731456820118/Put/seqid=0 2024-11-13T00:13:41,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741903_1079 (size=13017) 2024-11-13T00:13:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456881126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,131 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d 2024-11-13T00:13:41,138 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 7a0bbaa1ce124f8ea3c4ffbe5125f66d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:41,138 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:41,138 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456820638; duration=0sec 2024-11-13T00:13:41,139 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:41,139 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:41,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456881136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456881137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/2975749efdd34bbd9d9d65dc95b3b11f 2024-11-13T00:13:41,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/79915a300ddd4d898480da19fc2d2345 is 50, key is test_row_0/B:col10/1731456820150/Put/seqid=0 2024-11-13T00:13:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741904_1080 (size=12301) 2024-11-13T00:13:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T00:13:41,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T00:13:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T00:13:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456881436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456881446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456881446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T00:13:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:41,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T00:13:41,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/79915a300ddd4d898480da19fc2d2345 2024-11-13T00:13:41,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/34865e59f9264ea79740f5d2e98802e7 is 50, key is test_row_0/C:col10/1731456820150/Put/seqid=0 2024-11-13T00:13:41,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741905_1081 (size=12301) 2024-11-13T00:13:41,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/34865e59f9264ea79740f5d2e98802e7 2024-11-13T00:13:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/2975749efdd34bbd9d9d65dc95b3b11f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2975749efdd34bbd9d9d65dc95b3b11f 2024-11-13T00:13:41,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2975749efdd34bbd9d9d65dc95b3b11f, entries=200, sequenceid=320, filesize=14.4 K 2024-11-13T00:13:41,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/79915a300ddd4d898480da19fc2d2345 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/79915a300ddd4d898480da19fc2d2345 2024-11-13T00:13:41,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/79915a300ddd4d898480da19fc2d2345, entries=150, sequenceid=320, filesize=12.0 K 2024-11-13T00:13:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/34865e59f9264ea79740f5d2e98802e7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34865e59f9264ea79740f5d2e98802e7 2024-11-13T00:13:41,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34865e59f9264ea79740f5d2e98802e7, entries=150, sequenceid=320, filesize=12.0 K 2024-11-13T00:13:41,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 881ms, sequenceid=320, compaction requested=false 2024-11-13T00:13:41,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:41,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T00:13:41,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:41,719 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:13:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:41,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ac4fb64b7c3040999d90c2170ce934fa is 50, key is test_row_0/A:col10/1731456820809/Put/seqid=0 2024-11-13T00:13:41,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741906_1082 (size=12301) 2024-11-13T00:13:41,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:41,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456881983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456881984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456881986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T00:13:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456882096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456882097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456882097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,135 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ac4fb64b7c3040999d90c2170ce934fa 2024-11-13T00:13:42,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/57031257478843afb8da6690bc61dd1f is 50, key is test_row_0/B:col10/1731456820809/Put/seqid=0 2024-11-13T00:13:42,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741907_1083 (size=12301) 2024-11-13T00:13:42,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456882306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456882306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456882306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,560 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/57031257478843afb8da6690bc61dd1f 2024-11-13T00:13:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/9a08a61fd11f47ab9dcf4a1cd37ab367 is 50, key is test_row_0/C:col10/1731456820809/Put/seqid=0 2024-11-13T00:13:42,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741908_1084 (size=12301) 2024-11-13T00:13:42,608 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/9a08a61fd11f47ab9dcf4a1cd37ab367 2024-11-13T00:13:42,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456882616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456882616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456882616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ac4fb64b7c3040999d90c2170ce934fa as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac4fb64b7c3040999d90c2170ce934fa 2024-11-13T00:13:42,640 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac4fb64b7c3040999d90c2170ce934fa, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T00:13:42,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/57031257478843afb8da6690bc61dd1f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/57031257478843afb8da6690bc61dd1f 2024-11-13T00:13:42,655 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/57031257478843afb8da6690bc61dd1f, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T00:13:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/9a08a61fd11f47ab9dcf4a1cd37ab367 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9a08a61fd11f47ab9dcf4a1cd37ab367 2024-11-13T00:13:42,667 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9a08a61fd11f47ab9dcf4a1cd37ab367, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T00:13:42,670 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 883bdd7db86faeb1b8a9722c1fd955c1 in 951ms, sequenceid=335, compaction requested=true 2024-11-13T00:13:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-13T00:13:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-13T00:13:42,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T00:13:42,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-13T00:13:42,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7410 sec 2024-11-13T00:13:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T00:13:43,086 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:43,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-13T00:13:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T00:13:43,089 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:43,090 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:43,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:43,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T00:13:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:43,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d1295989b7a9435c88b9580e8799d01d is 50, key is test_row_0/A:col10/1731456821984/Put/seqid=0 2024-11-13T00:13:43,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741909_1085 (size=14741) 2024-11-13T00:13:43,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456883147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456883149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456883149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T00:13:43,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T00:13:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456883256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456883256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456883267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T00:13:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T00:13:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456883476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456883476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456883477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d1295989b7a9435c88b9580e8799d01d 2024-11-13T00:13:43,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T00:13:43,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:43,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:43,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/eb58fb61894340cf8640fd02ac09668d is 50, key is test_row_0/B:col10/1731456821984/Put/seqid=0 2024-11-13T00:13:43,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741910_1086 (size=12301) 2024-11-13T00:13:43,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/eb58fb61894340cf8640fd02ac09668d 2024-11-13T00:13:43,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/9b56cd96ce4c489a9e12876ba5cda08c is 50, key is test_row_0/C:col10/1731456821984/Put/seqid=0 2024-11-13T00:13:43,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741911_1087 (size=12301) 2024-11-13T00:13:43,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/9b56cd96ce4c489a9e12876ba5cda08c 2024-11-13T00:13:43,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d1295989b7a9435c88b9580e8799d01d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1295989b7a9435c88b9580e8799d01d 2024-11-13T00:13:43,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1295989b7a9435c88b9580e8799d01d, entries=200, sequenceid=356, filesize=14.4 K 2024-11-13T00:13:43,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/eb58fb61894340cf8640fd02ac09668d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/eb58fb61894340cf8640fd02ac09668d 2024-11-13T00:13:43,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/eb58fb61894340cf8640fd02ac09668d, entries=150, sequenceid=356, filesize=12.0 K 2024-11-13T00:13:43,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/9b56cd96ce4c489a9e12876ba5cda08c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9b56cd96ce4c489a9e12876ba5cda08c 2024-11-13T00:13:43,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9b56cd96ce4c489a9e12876ba5cda08c, entries=150, sequenceid=356, filesize=12.0 K 2024-11-13T00:13:43,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 883bdd7db86faeb1b8a9722c1fd955c1 in 532ms, sequenceid=356, compaction requested=true 2024-11-13T00:13:43,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:43,660 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:43,661 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:43,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:43,662 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:43,662 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:43,662 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,663 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/a07e938e61234566b60351de9ca1d0a2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2975749efdd34bbd9d9d65dc95b3b11f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac4fb64b7c3040999d90c2170ce934fa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1295989b7a9435c88b9580e8799d01d] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=53.5 K 2024-11-13T00:13:43,665 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:43,665 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:43,665 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,666 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/79915a300ddd4d898480da19fc2d2345, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/57031257478843afb8da6690bc61dd1f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/eb58fb61894340cf8640fd02ac09668d] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=48.8 K 2024-11-13T00:13:43,666 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07e938e61234566b60351de9ca1d0a2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731456819481 2024-11-13T00:13:43,667 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting db5cdeb4c3d84ec48ba9265d8e9ac5d0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731456819481 2024-11-13T00:13:43,667 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2975749efdd34bbd9d9d65dc95b3b11f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731456820150 2024-11-13T00:13:43,667 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac4fb64b7c3040999d90c2170ce934fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731456820801 2024-11-13T00:13:43,667 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 79915a300ddd4d898480da19fc2d2345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731456820150 2024-11-13T00:13:43,668 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1295989b7a9435c88b9580e8799d01d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731456821970 2024-11-13T00:13:43,668 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 57031257478843afb8da6690bc61dd1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731456820801 2024-11-13T00:13:43,669 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting eb58fb61894340cf8640fd02ac09668d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731456821982 2024-11-13T00:13:43,683 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:43,684 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1fae5993d6464a61abbb3a30030e34c6 is 50, key is test_row_0/B:col10/1731456821984/Put/seqid=0 2024-11-13T00:13:43,691 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:43,691 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d3ab12d81ab343edb767624960d17bc1 is 50, key is test_row_0/A:col10/1731456821984/Put/seqid=0 2024-11-13T00:13:43,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741912_1088 (size=13153) 2024-11-13T00:13:43,715 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1fae5993d6464a61abbb3a30030e34c6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1fae5993d6464a61abbb3a30030e34c6 2024-11-13T00:13:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T00:13:43,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T00:13:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,718 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:13:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:43,727 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into 1fae5993d6464a61abbb3a30030e34c6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:43,728 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:43,728 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=12, startTime=1731456823661; duration=0sec 2024-11-13T00:13:43,728 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:43,728 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:43,728 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:13:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741913_1089 (size=13153) 2024-11-13T00:13:43,732 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:13:43,732 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:43,732 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/bfa6d507649e4a9ca6471a5e19427307 is 50, key is test_row_0/A:col10/1731456823141/Put/seqid=0 2024-11-13T00:13:43,736 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34865e59f9264ea79740f5d2e98802e7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9a08a61fd11f47ab9dcf4a1cd37ab367, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9b56cd96ce4c489a9e12876ba5cda08c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=48.8 K 2024-11-13T00:13:43,738 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a0bbaa1ce124f8ea3c4ffbe5125f66d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731456819481 2024-11-13T00:13:43,740 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/d3ab12d81ab343edb767624960d17bc1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d3ab12d81ab343edb767624960d17bc1 2024-11-13T00:13:43,740 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 34865e59f9264ea79740f5d2e98802e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731456820150 2024-11-13T00:13:43,741 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a08a61fd11f47ab9dcf4a1cd37ab367, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731456820801 2024-11-13T00:13:43,741 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b56cd96ce4c489a9e12876ba5cda08c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731456821982 2024-11-13T00:13:43,748 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into d3ab12d81ab343edb767624960d17bc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:43,748 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:43,748 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=12, startTime=1731456823660; duration=0sec 2024-11-13T00:13:43,748 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:43,748 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741914_1090 (size=12301) 2024-11-13T00:13:43,776 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/bfa6d507649e4a9ca6471a5e19427307 2024-11-13T00:13:43,779 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:43,781 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/4ceaafd18773417fa69aca26dae31f5e is 50, key is test_row_0/C:col10/1731456821984/Put/seqid=0 2024-11-13T00:13:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/421c3f62553a4a5ba6fb5d81573775e4 is 50, key is test_row_0/B:col10/1731456823141/Put/seqid=0 2024-11-13T00:13:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741916_1092 (size=12301) 2024-11-13T00:13:43,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741915_1091 (size=13153) 2024-11-13T00:13:43,827 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/4ceaafd18773417fa69aca26dae31f5e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/4ceaafd18773417fa69aca26dae31f5e 2024-11-13T00:13:43,848 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 4ceaafd18773417fa69aca26dae31f5e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:43,848 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:43,848 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=12, startTime=1731456823661; duration=0sec 2024-11-13T00:13:43,849 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:43,849 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456883842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456823832, max timestamp=1731456823846Num uniques 859; 2024-11-13T00:13:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:13:43,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456823832, max timestamp=1731456823846Num uniques 859; 2024-11-13T00:13:43,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:13:43,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456823835, max timestamp=1731456823846Num uniques 859; 2024-11-13T00:13:43,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:13:43,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456883953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456883956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456883961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456884065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456884076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456884166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,208 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/421c3f62553a4a5ba6fb5d81573775e4 2024-11-13T00:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T00:13:44,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/6acf4908ecac45b0ae9208c142cf536a is 50, key is test_row_0/C:col10/1731456823141/Put/seqid=0 2024-11-13T00:13:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741917_1093 (size=12301) 2024-11-13T00:13:44,271 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/6acf4908ecac45b0ae9208c142cf536a 2024-11-13T00:13:44,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/bfa6d507649e4a9ca6471a5e19427307 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bfa6d507649e4a9ca6471a5e19427307 2024-11-13T00:13:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456884275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,285 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bfa6d507649e4a9ca6471a5e19427307, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T00:13:44,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/421c3f62553a4a5ba6fb5d81573775e4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/421c3f62553a4a5ba6fb5d81573775e4 2024-11-13T00:13:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456884286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,294 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/421c3f62553a4a5ba6fb5d81573775e4, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T00:13:44,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/6acf4908ecac45b0ae9208c142cf536a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/6acf4908ecac45b0ae9208c142cf536a 2024-11-13T00:13:44,312 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/6acf4908ecac45b0ae9208c142cf536a, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T00:13:44,313 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 883bdd7db86faeb1b8a9722c1fd955c1 in 595ms, sequenceid=367, compaction requested=false 2024-11-13T00:13:44,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:44,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-13T00:13:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-13T00:13:44,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T00:13:44,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2250 sec 2024-11-13T00:13:44,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2350 sec 2024-11-13T00:13:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:44,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-13T00:13:44,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:44,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:44,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:44,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:44,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:44,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:44,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/fca4d74b711d4ddc8e4e8b6cced06440 is 50, key is test_row_0/A:col10/1731456823958/Put/seqid=0 2024-11-13T00:13:44,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741918_1094 (size=12301) 2024-11-13T00:13:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456884538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456884586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456884595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456884646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456884856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:44,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:44,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/fca4d74b711d4ddc8e4e8b6cced06440 2024-11-13T00:13:44,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/b441a0237a994150b1b7c9b1d10d6b74 is 50, key is test_row_0/B:col10/1731456823958/Put/seqid=0 2024-11-13T00:13:44,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741919_1095 (size=12301) 2024-11-13T00:13:45,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456885096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:45,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456885111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:45,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456885181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T00:13:45,237 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:45,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-13T00:13:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T00:13:45,242 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:45,243 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:45,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T00:13:45,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T00:13:45,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:45,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/b441a0237a994150b1b7c9b1d10d6b74 2024-11-13T00:13:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/51b2a0ee95a645529e24422d3e34d910 is 50, key is test_row_0/C:col10/1731456823958/Put/seqid=0 2024-11-13T00:13:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741920_1096 (size=12301) 2024-11-13T00:13:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T00:13:45,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T00:13:45,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456885696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:45,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T00:13:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/51b2a0ee95a645529e24422d3e34d910 2024-11-13T00:13:45,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/fca4d74b711d4ddc8e4e8b6cced06440 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fca4d74b711d4ddc8e4e8b6cced06440 2024-11-13T00:13:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T00:13:45,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T00:13:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:45,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fca4d74b711d4ddc8e4e8b6cced06440, entries=150, sequenceid=406, filesize=12.0 K 2024-11-13T00:13:45,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/b441a0237a994150b1b7c9b1d10d6b74 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b441a0237a994150b1b7c9b1d10d6b74 2024-11-13T00:13:45,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b441a0237a994150b1b7c9b1d10d6b74, entries=150, sequenceid=406, filesize=12.0 K 2024-11-13T00:13:45,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/51b2a0ee95a645529e24422d3e34d910 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/51b2a0ee95a645529e24422d3e34d910 2024-11-13T00:13:45,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/51b2a0ee95a645529e24422d3e34d910, entries=150, sequenceid=406, filesize=12.0 K 2024-11-13T00:13:45,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 883bdd7db86faeb1b8a9722c1fd955c1 in 1410ms, sequenceid=406, compaction requested=true 2024-11-13T00:13:45,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:45,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:45,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:45,894 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:45,894 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:45,897 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:45,898 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:45,898 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,898 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1fae5993d6464a61abbb3a30030e34c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/421c3f62553a4a5ba6fb5d81573775e4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b441a0237a994150b1b7c9b1d10d6b74] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=36.9 K 2024-11-13T00:13:45,898 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:45,898 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:45,898 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,899 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d3ab12d81ab343edb767624960d17bc1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bfa6d507649e4a9ca6471a5e19427307, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fca4d74b711d4ddc8e4e8b6cced06440] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=36.9 K 2024-11-13T00:13:45,899 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fae5993d6464a61abbb3a30030e34c6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731456821982 2024-11-13T00:13:45,899 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 421c3f62553a4a5ba6fb5d81573775e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456823141 2024-11-13T00:13:45,900 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ab12d81ab343edb767624960d17bc1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731456821982 2024-11-13T00:13:45,900 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b441a0237a994150b1b7c9b1d10d6b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731456823950 2024-11-13T00:13:45,900 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa6d507649e4a9ca6471a5e19427307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456823141 2024-11-13T00:13:45,901 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca4d74b711d4ddc8e4e8b6cced06440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731456823950 2024-11-13T00:13:45,925 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:45,926 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/db265ade19324f7f86bced5a1cb99ced is 50, key is test_row_0/B:col10/1731456823958/Put/seqid=0 2024-11-13T00:13:45,934 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:45,934 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0df6c598525b4631801735b739293007 is 50, key is test_row_0/A:col10/1731456823958/Put/seqid=0 2024-11-13T00:13:45,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741921_1097 (size=13255) 2024-11-13T00:13:45,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741922_1098 (size=13255) 2024-11-13T00:13:45,976 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0df6c598525b4631801735b739293007 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0df6c598525b4631801735b739293007 2024-11-13T00:13:45,982 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/db265ade19324f7f86bced5a1cb99ced as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db265ade19324f7f86bced5a1cb99ced 2024-11-13T00:13:45,991 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into db265ade19324f7f86bced5a1cb99ced(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:45,991 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:45,991 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456825894; duration=0sec 2024-11-13T00:13:45,991 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:45,991 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:45,992 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:45,992 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 0df6c598525b4631801735b739293007(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:45,993 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:45,993 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:45,993 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:45,993 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/4ceaafd18773417fa69aca26dae31f5e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/6acf4908ecac45b0ae9208c142cf536a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/51b2a0ee95a645529e24422d3e34d910] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=36.9 K 2024-11-13T00:13:45,994 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:45,994 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456825893; duration=0sec 2024-11-13T00:13:45,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ceaafd18773417fa69aca26dae31f5e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731456821982 2024-11-13T00:13:45,994 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:45,994 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:45,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6acf4908ecac45b0ae9208c142cf536a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456823141 2024-11-13T00:13:45,995 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 51b2a0ee95a645529e24422d3e34d910, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731456823950 2024-11-13T00:13:46,012 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:46,013 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/2c91dd2f294647afb11942d5241ed55d is 50, key is test_row_0/C:col10/1731456823958/Put/seqid=0 2024-11-13T00:13:46,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T00:13:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,023 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:13:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:46,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/bc93b430f55a47c692277eb6bc5fcb51 is 50, key is test_row_0/A:col10/1731456824532/Put/seqid=0 2024-11-13T00:13:46,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741923_1099 (size=13255) 2024-11-13T00:13:46,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741924_1100 (size=12301) 2024-11-13T00:13:46,077 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/bc93b430f55a47c692277eb6bc5fcb51 2024-11-13T00:13:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/3257e78cd84f4f9a82854e990866921f is 50, key is test_row_0/B:col10/1731456824532/Put/seqid=0 2024-11-13T00:13:46,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741925_1101 (size=12301) 2024-11-13T00:13:46,109 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/3257e78cd84f4f9a82854e990866921f 2024-11-13T00:13:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:46,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/935ff3c17d0242c1b6a4a420867430e3 is 50, key is test_row_0/C:col10/1731456824532/Put/seqid=0 2024-11-13T00:13:46,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741926_1102 (size=12301) 2024-11-13T00:13:46,156 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/935ff3c17d0242c1b6a4a420867430e3 2024-11-13T00:13:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/bc93b430f55a47c692277eb6bc5fcb51 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bc93b430f55a47c692277eb6bc5fcb51 2024-11-13T00:13:46,173 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bc93b430f55a47c692277eb6bc5fcb51, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T00:13:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/3257e78cd84f4f9a82854e990866921f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/3257e78cd84f4f9a82854e990866921f 2024-11-13T00:13:46,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456826156, max timestamp=1731456826173Num uniques 859; 2024-11-13T00:13:46,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456826156, max timestamp=1731456826173Num uniques 859; 2024-11-13T00:13:46,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:13:46,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:13:46,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456826156, max timestamp=1731456826173Num uniques 859; 2024-11-13T00:13:46,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:13:46,183 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/3257e78cd84f4f9a82854e990866921f, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T00:13:46,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/935ff3c17d0242c1b6a4a420867430e3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/935ff3c17d0242c1b6a4a420867430e3 2024-11-13T00:13:46,198 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/935ff3c17d0242c1b6a4a420867430e3, entries=150, sequenceid=418, filesize=12.0 K 2024-11-13T00:13:46,199 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=87.22 KB/89310 for 883bdd7db86faeb1b8a9722c1fd955c1 in 176ms, sequenceid=418, compaction requested=false 2024-11-13T00:13:46,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:46,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-13T00:13:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-13T00:13:46,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=93.93 KB heapSize=256.08 KB 2024-11-13T00:13:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:13:46,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-13T00:13:46,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 959 msec 2024-11-13T00:13:46,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 970 msec 2024-11-13T00:13:46,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ce37dd22a45d46e79f3bcecda65e4fe7 is 50, key is test_row_0/A:col10/1731456826199/Put/seqid=0 2024-11-13T00:13:46,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741927_1103 (size=22065) 2024-11-13T00:13:46,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ce37dd22a45d46e79f3bcecda65e4fe7 2024-11-13T00:13:46,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9d2bec4da8f14f68a67746c8e052cbf2 is 50, key is test_row_0/B:col10/1731456826199/Put/seqid=0 2024-11-13T00:13:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741928_1104 (size=12301) 2024-11-13T00:13:46,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9d2bec4da8f14f68a67746c8e052cbf2 2024-11-13T00:13:46,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456886256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/bea2699403414abb95ede1d40c3d60a3 is 50, key is test_row_0/C:col10/1731456826199/Put/seqid=0 2024-11-13T00:13:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456886266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741929_1105 (size=12301) 2024-11-13T00:13:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T00:13:46,376 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:46,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:46,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456886376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-13T00:13:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T00:13:46,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:46,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:46,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456886386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,473 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/2c91dd2f294647afb11942d5241ed55d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/2c91dd2f294647afb11942d5241ed55d 2024-11-13T00:13:46,485 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 2c91dd2f294647afb11942d5241ed55d(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:46,485 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:46,485 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456825894; duration=0sec 2024-11-13T00:13:46,486 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:46,486 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T00:13:46,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T00:13:46,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:46,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:46,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456886588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456886598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/bea2699403414abb95ede1d40c3d60a3 2024-11-13T00:13:46,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T00:13:46,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:46,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:46,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T00:13:46,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/ce37dd22a45d46e79f3bcecda65e4fe7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ce37dd22a45d46e79f3bcecda65e4fe7 2024-11-13T00:13:46,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ce37dd22a45d46e79f3bcecda65e4fe7, entries=350, sequenceid=447, filesize=21.5 K 2024-11-13T00:13:46,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9d2bec4da8f14f68a67746c8e052cbf2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9d2bec4da8f14f68a67746c8e052cbf2 2024-11-13T00:13:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456886716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,721 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9d2bec4da8f14f68a67746c8e052cbf2, entries=150, sequenceid=447, filesize=12.0 K 2024-11-13T00:13:46,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/bea2699403414abb95ede1d40c3d60a3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/bea2699403414abb95ede1d40c3d60a3 2024-11-13T00:13:46,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/bea2699403414abb95ede1d40c3d60a3, entries=150, sequenceid=447, filesize=12.0 K 2024-11-13T00:13:46,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~256.03 KB/262176, currentSize=87.22 KB/89310 for 883bdd7db86faeb1b8a9722c1fd955c1 in 542ms, sequenceid=447, compaction requested=true 2024-11-13T00:13:46,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:46,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:46,743 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:46,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:46,743 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:46,744 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:46,745 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:46,745 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,745 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db265ade19324f7f86bced5a1cb99ced, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/3257e78cd84f4f9a82854e990866921f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9d2bec4da8f14f68a67746c8e052cbf2] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.0 K 2024-11-13T00:13:46,745 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47621 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:46,745 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting db265ade19324f7f86bced5a1cb99ced, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731456823950 2024-11-13T00:13:46,745 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:46,746 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,746 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0df6c598525b4631801735b739293007, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bc93b430f55a47c692277eb6bc5fcb51, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ce37dd22a45d46e79f3bcecda65e4fe7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=46.5 K 2024-11-13T00:13:46,746 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3257e78cd84f4f9a82854e990866921f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731456824512 2024-11-13T00:13:46,746 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df6c598525b4631801735b739293007, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731456823950 2024-11-13T00:13:46,747 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2bec4da8f14f68a67746c8e052cbf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731456826189 2024-11-13T00:13:46,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:46,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:46,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:46,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:46,747 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc93b430f55a47c692277eb6bc5fcb51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731456824512 2024-11-13T00:13:46,748 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce37dd22a45d46e79f3bcecda65e4fe7, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731456826173 2024-11-13T00:13:46,761 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:46,763 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:46,764 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/89d85c313b294cc58af8281aaf736cf6 is 50, key is test_row_0/A:col10/1731456826199/Put/seqid=0 2024-11-13T00:13:46,766 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/4a27a3014cd64f3383bcd720eb64b883 is 50, key is test_row_0/B:col10/1731456826199/Put/seqid=0 2024-11-13T00:13:46,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741930_1106 (size=13357) 2024-11-13T00:13:46,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741931_1107 (size=13357) 2024-11-13T00:13:46,822 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/4a27a3014cd64f3383bcd720eb64b883 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/4a27a3014cd64f3383bcd720eb64b883 2024-11-13T00:13:46,829 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into 4a27a3014cd64f3383bcd720eb64b883(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:46,829 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:46,829 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456826743; duration=0sec 2024-11-13T00:13:46,830 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:46,830 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:46,830 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:46,832 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:46,833 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:46,833 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,833 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/2c91dd2f294647afb11942d5241ed55d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/935ff3c17d0242c1b6a4a420867430e3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/bea2699403414abb95ede1d40c3d60a3] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.0 K 2024-11-13T00:13:46,834 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c91dd2f294647afb11942d5241ed55d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731456823950 2024-11-13T00:13:46,834 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 935ff3c17d0242c1b6a4a420867430e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731456824512 2024-11-13T00:13:46,835 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bea2699403414abb95ede1d40c3d60a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731456826189 2024-11-13T00:13:46,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T00:13:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:46,844 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T00:13:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:46,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:46,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/41f1155ac60947998fbe4c6fb74aad78 is 50, key is test_row_0/A:col10/1731456826252/Put/seqid=0 2024-11-13T00:13:46,880 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:46,880 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/01b8c569e19c4b4c84f8769ebaea5b74 is 50, key is test_row_0/C:col10/1731456826199/Put/seqid=0 2024-11-13T00:13:46,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741932_1108 (size=12301) 2024-11-13T00:13:46,888 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/41f1155ac60947998fbe4c6fb74aad78 2024-11-13T00:13:46,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/8e29adfcfeb7457498e98f106cec5d2d is 50, key is test_row_0/B:col10/1731456826252/Put/seqid=0 2024-11-13T00:13:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:46,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741933_1109 (size=13357) 2024-11-13T00:13:46,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741934_1110 (size=12301) 2024-11-13T00:13:46,933 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/01b8c569e19c4b4c84f8769ebaea5b74 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/01b8c569e19c4b4c84f8769ebaea5b74 2024-11-13T00:13:46,940 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/8e29adfcfeb7457498e98f106cec5d2d 2024-11-13T00:13:46,947 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 01b8c569e19c4b4c84f8769ebaea5b74(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:46,949 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:46,949 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456826747; duration=0sec 2024-11-13T00:13:46,950 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:46,950 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:46,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/7c7c4d84868b47b2b77cc1c4af0ce780 is 50, key is test_row_0/C:col10/1731456826252/Put/seqid=0 2024-11-13T00:13:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456886967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456886970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:46,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741935_1111 (size=12301) 2024-11-13T00:13:47,004 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/7c7c4d84868b47b2b77cc1c4af0ce780 2024-11-13T00:13:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T00:13:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/41f1155ac60947998fbe4c6fb74aad78 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/41f1155ac60947998fbe4c6fb74aad78 2024-11-13T00:13:47,024 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/41f1155ac60947998fbe4c6fb74aad78, entries=150, sequenceid=465, filesize=12.0 K 2024-11-13T00:13:47,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/8e29adfcfeb7457498e98f106cec5d2d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/8e29adfcfeb7457498e98f106cec5d2d 2024-11-13T00:13:47,034 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/8e29adfcfeb7457498e98f106cec5d2d, entries=150, sequenceid=465, filesize=12.0 K 2024-11-13T00:13:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/7c7c4d84868b47b2b77cc1c4af0ce780 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7c7c4d84868b47b2b77cc1c4af0ce780 2024-11-13T00:13:47,054 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7c7c4d84868b47b2b77cc1c4af0ce780, entries=150, sequenceid=465, filesize=12.0 K 2024-11-13T00:13:47,056 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 883bdd7db86faeb1b8a9722c1fd955c1 in 211ms, sequenceid=465, compaction requested=false 2024-11-13T00:13:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-13T00:13:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-13T00:13:47,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T00:13:47,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 684 msec 2024-11-13T00:13:47,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 692 msec 2024-11-13T00:13:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:47,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T00:13:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:47,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0fc5c81586c344dea4a47f6a4fc2cadf is 50, key is test_row_0/A:col10/1731456827090/Put/seqid=0 2024-11-13T00:13:47,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741936_1112 (size=12301) 2024-11-13T00:13:47,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0fc5c81586c344dea4a47f6a4fc2cadf 2024-11-13T00:13:47,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/87b2cc6e3f4d47da9975646514434433 is 50, key is test_row_0/B:col10/1731456827090/Put/seqid=0 2024-11-13T00:13:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456887138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456887141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741937_1113 (size=12301) 2024-11-13T00:13:47,202 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/89d85c313b294cc58af8281aaf736cf6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/89d85c313b294cc58af8281aaf736cf6 2024-11-13T00:13:47,208 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 89d85c313b294cc58af8281aaf736cf6(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:47,208 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:47,208 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456826743; duration=0sec 2024-11-13T00:13:47,208 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:47,208 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:47,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456887256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456887257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456887466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456887466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T00:13:47,516 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:47,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-13T00:13:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T00:13:47,519 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:47,520 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:47,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:47,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/87b2cc6e3f4d47da9975646514434433 2024-11-13T00:13:47,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/5fb487575aa34cc8bb11949b58cc7baf is 50, key is test_row_0/C:col10/1731456827090/Put/seqid=0 2024-11-13T00:13:47,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741938_1114 (size=12301) 2024-11-13T00:13:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T00:13:47,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T00:13:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456887776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456887777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:47,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T00:13:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T00:13:47,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T00:13:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:47,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:47,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/5fb487575aa34cc8bb11949b58cc7baf 2024-11-13T00:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/0fc5c81586c344dea4a47f6a4fc2cadf as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0fc5c81586c344dea4a47f6a4fc2cadf 2024-11-13T00:13:48,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0fc5c81586c344dea4a47f6a4fc2cadf, entries=150, sequenceid=485, filesize=12.0 K 2024-11-13T00:13:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/87b2cc6e3f4d47da9975646514434433 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/87b2cc6e3f4d47da9975646514434433 2024-11-13T00:13:48,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/87b2cc6e3f4d47da9975646514434433, entries=150, sequenceid=485, filesize=12.0 K 2024-11-13T00:13:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/5fb487575aa34cc8bb11949b58cc7baf as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/5fb487575aa34cc8bb11949b58cc7baf 2024-11-13T00:13:48,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/5fb487575aa34cc8bb11949b58cc7baf, entries=150, sequenceid=485, filesize=12.0 K 2024-11-13T00:13:48,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 883bdd7db86faeb1b8a9722c1fd955c1 in 931ms, sequenceid=485, compaction requested=true 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:48,022 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:48,022 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:48,024 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:48,024 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:48,024 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:48,024 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:48,024 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:48,024 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:48,024 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/89d85c313b294cc58af8281aaf736cf6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/41f1155ac60947998fbe4c6fb74aad78, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0fc5c81586c344dea4a47f6a4fc2cadf] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.1 K 2024-11-13T00:13:48,024 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/4a27a3014cd64f3383bcd720eb64b883, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/8e29adfcfeb7457498e98f106cec5d2d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/87b2cc6e3f4d47da9975646514434433] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.1 K 2024-11-13T00:13:48,025 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d85c313b294cc58af8281aaf736cf6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731456826189 2024-11-13T00:13:48,025 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a27a3014cd64f3383bcd720eb64b883, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731456826189 2024-11-13T00:13:48,026 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41f1155ac60947998fbe4c6fb74aad78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731456826252 2024-11-13T00:13:48,026 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e29adfcfeb7457498e98f106cec5d2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731456826252 2024-11-13T00:13:48,026 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc5c81586c344dea4a47f6a4fc2cadf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731456826939 2024-11-13T00:13:48,026 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b2cc6e3f4d47da9975646514434433, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731456826939 2024-11-13T00:13:48,044 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:48,044 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/5f28db9eba0046168fa3be76c95d81fd is 50, key is test_row_0/B:col10/1731456827090/Put/seqid=0 2024-11-13T00:13:48,044 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:48,045 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/710ff2d973ff495486814cd2a7919a6d is 50, key is test_row_0/A:col10/1731456827090/Put/seqid=0 2024-11-13T00:13:48,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741939_1115 (size=13459) 2024-11-13T00:13:48,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741940_1116 (size=13459) 2024-11-13T00:13:48,073 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/710ff2d973ff495486814cd2a7919a6d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/710ff2d973ff495486814cd2a7919a6d 2024-11-13T00:13:48,084 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 710ff2d973ff495486814cd2a7919a6d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:48,084 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:48,084 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456828022; duration=0sec 2024-11-13T00:13:48,085 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:48,085 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:48,085 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:48,086 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:48,086 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:48,086 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:48,086 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/01b8c569e19c4b4c84f8769ebaea5b74, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7c7c4d84868b47b2b77cc1c4af0ce780, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/5fb487575aa34cc8bb11949b58cc7baf] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.1 K 2024-11-13T00:13:48,087 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01b8c569e19c4b4c84f8769ebaea5b74, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731456826189 2024-11-13T00:13:48,087 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7c4d84868b47b2b77cc1c4af0ce780, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731456826252 2024-11-13T00:13:48,088 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb487575aa34cc8bb11949b58cc7baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731456826939 2024-11-13T00:13:48,100 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:48,101 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/49e3fc73a6f8421c999709f21efc9d4c is 50, key is test_row_0/C:col10/1731456827090/Put/seqid=0 2024-11-13T00:13:48,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741941_1117 (size=13459) 2024-11-13T00:13:48,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T00:13:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:48,136 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:13:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T00:13:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/8428792b69bb46cabac6721995237659 is 50, key is test_row_0/A:col10/1731456827138/Put/seqid=0 2024-11-13T00:13:48,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741942_1118 (size=12301) 2024-11-13T00:13:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:48,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456888321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456888327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456888436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456888436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,463 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/5f28db9eba0046168fa3be76c95d81fd as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5f28db9eba0046168fa3be76c95d81fd 2024-11-13T00:13:48,472 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into 5f28db9eba0046168fa3be76c95d81fd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:48,472 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:48,472 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456828022; duration=0sec 2024-11-13T00:13:48,472 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:48,472 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:48,515 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/49e3fc73a6f8421c999709f21efc9d4c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/49e3fc73a6f8421c999709f21efc9d4c 2024-11-13T00:13:48,528 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 49e3fc73a6f8421c999709f21efc9d4c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:48,528 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:48,528 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456828022; duration=0sec 2024-11-13T00:13:48,529 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:48,529 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:48,561 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/8428792b69bb46cabac6721995237659 2024-11-13T00:13:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9ae5188d0dea4902a84b15e55919c043 is 50, key is test_row_0/B:col10/1731456827138/Put/seqid=0 2024-11-13T00:13:48,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741943_1119 (size=12301) 2024-11-13T00:13:48,579 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9ae5188d0dea4902a84b15e55919c043 2024-11-13T00:13:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/c44e83e64a7c4769a1ba075a2d663df0 is 50, key is test_row_0/C:col10/1731456827138/Put/seqid=0 2024-11-13T00:13:48,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741944_1120 (size=12301) 2024-11-13T00:13:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456888646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456888646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T00:13:48,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47400 deadline: 1731456888746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,750 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456888956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456888956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:48,996 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/c44e83e64a7c4769a1ba075a2d663df0 2024-11-13T00:13:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/8428792b69bb46cabac6721995237659 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/8428792b69bb46cabac6721995237659 2024-11-13T00:13:49,008 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/8428792b69bb46cabac6721995237659, entries=150, sequenceid=500, filesize=12.0 K 2024-11-13T00:13:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/9ae5188d0dea4902a84b15e55919c043 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9ae5188d0dea4902a84b15e55919c043 2024-11-13T00:13:49,014 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9ae5188d0dea4902a84b15e55919c043, entries=150, sequenceid=500, filesize=12.0 K 2024-11-13T00:13:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/c44e83e64a7c4769a1ba075a2d663df0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/c44e83e64a7c4769a1ba075a2d663df0 2024-11-13T00:13:49,021 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/c44e83e64a7c4769a1ba075a2d663df0, entries=150, sequenceid=500, filesize=12.0 K 2024-11-13T00:13:49,025 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 883bdd7db86faeb1b8a9722c1fd955c1 in 889ms, sequenceid=500, compaction requested=false 2024-11-13T00:13:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-13T00:13:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-13T00:13:49,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T00:13:49,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-13T00:13:49,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.5150 sec 2024-11-13T00:13:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:49,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:49,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3c3bdbef6f614feab658605dcd85add3 is 50, key is test_row_0/A:col10/1731456828310/Put/seqid=0 2024-11-13T00:13:49,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741945_1121 (size=14741) 2024-11-13T00:13:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456889530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:49,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456889531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456889636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456889646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T00:13:49,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:13:49,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:13:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-13T00:13:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:49,669 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:13:49,670 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:13:49,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:13:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:49,805 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T00:13:49,805 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T00:13:49,805 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:49,805 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:49,806 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T00:13:49,806 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:49,808 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T00:13:49,808 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:49,810 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,810 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,810 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,810 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,811 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,811 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:49,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:49,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:49,811 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:49,811 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:49,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:49,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:49,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:49,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456889846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:49,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456889856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:49,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3c3bdbef6f614feab658605dcd85add3 2024-11-13T00:13:49,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/22ea575447084db481492870e513f65b is 50, key is test_row_0/B:col10/1731456828310/Put/seqid=0 2024-11-13T00:13:49,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741946_1122 (size=12301) 2024-11-13T00:13:49,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:49,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:50,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:50,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:50,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456890155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456890166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:50,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:50,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:50,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:50,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1731456890296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:50,296 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:50,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/22ea575447084db481492870e513f65b 2024-11-13T00:13:50,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/b7480eedaf0349ab9a3f91c2011821b8 is 50, key is test_row_0/C:col10/1731456828310/Put/seqid=0 2024-11-13T00:13:50,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741947_1123 (size=12301) 2024-11-13T00:13:50,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47370 deadline: 1731456890336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:50,337 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 883bdd7db86faeb1b8a9722c1fd955c1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:50,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:50,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:50,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47390 deadline: 1731456890665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:13:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47334 deadline: 1731456890676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:13:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:13:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:13:50,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/b7480eedaf0349ab9a3f91c2011821b8 2024-11-13T00:13:50,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3c3bdbef6f614feab658605dcd85add3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3c3bdbef6f614feab658605dcd85add3 2024-11-13T00:13:50,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3c3bdbef6f614feab658605dcd85add3, entries=200, sequenceid=522, filesize=14.4 K 2024-11-13T00:13:50,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/22ea575447084db481492870e513f65b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/22ea575447084db481492870e513f65b 2024-11-13T00:13:50,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/22ea575447084db481492870e513f65b, entries=150, sequenceid=522, filesize=12.0 K 2024-11-13T00:13:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:13:50,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/b7480eedaf0349ab9a3f91c2011821b8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/b7480eedaf0349ab9a3f91c2011821b8 2024-11-13T00:13:50,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/b7480eedaf0349ab9a3f91c2011821b8, entries=150, sequenceid=522, filesize=12.0 K 2024-11-13T00:13:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 883bdd7db86faeb1b8a9722c1fd955c1 in 1261ms, sequenceid=522, compaction requested=true 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:13:50,750 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 883bdd7db86faeb1b8a9722c1fd955c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:13:50,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:50,750 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:50,751 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:50,751 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:50,751 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/B is initiating minor compaction (all files) 2024-11-13T00:13:50,751 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/A is initiating minor compaction (all files) 2024-11-13T00:13:50,751 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/B in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,751 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/A in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,751 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/710ff2d973ff495486814cd2a7919a6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/8428792b69bb46cabac6721995237659, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3c3bdbef6f614feab658605dcd85add3] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=39.6 K 2024-11-13T00:13:50,751 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5f28db9eba0046168fa3be76c95d81fd, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9ae5188d0dea4902a84b15e55919c043, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/22ea575447084db481492870e513f65b] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.2 K 2024-11-13T00:13:50,751 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710ff2d973ff495486814cd2a7919a6d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731456826939 2024-11-13T00:13:50,751 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f28db9eba0046168fa3be76c95d81fd, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731456826939 2024-11-13T00:13:50,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8428792b69bb46cabac6721995237659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731456827126 2024-11-13T00:13:50,752 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae5188d0dea4902a84b15e55919c043, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731456827126 2024-11-13T00:13:50,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3bdbef6f614feab658605dcd85add3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731456828310 2024-11-13T00:13:50,752 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ea575447084db481492870e513f65b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731456828310 2024-11-13T00:13:50,760 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:50,760 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:50,761 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3fba868b02be48ef99622cd8486dff5a is 50, key is test_row_0/A:col10/1731456828310/Put/seqid=0 2024-11-13T00:13:50,761 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/5e746d1683a7421d9116a46376ed742d is 50, key is test_row_0/B:col10/1731456828310/Put/seqid=0 2024-11-13T00:13:50,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741948_1124 (size=13561) 2024-11-13T00:13:50,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741949_1125 (size=13561) 2024-11-13T00:13:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:50,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:50,897 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:13:50,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:13:50,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/4d069eee7be8404e8a2c951625dfbe11 is 50, key is test_row_1/A:col10/1731456829528/Put/seqid=0 2024-11-13T00:13:50,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741950_1126 (size=9857) 2024-11-13T00:13:51,172 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/5e746d1683a7421d9116a46376ed742d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5e746d1683a7421d9116a46376ed742d 2024-11-13T00:13:51,172 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/3fba868b02be48ef99622cd8486dff5a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3fba868b02be48ef99622cd8486dff5a 2024-11-13T00:13:51,177 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/A of 883bdd7db86faeb1b8a9722c1fd955c1 into 3fba868b02be48ef99622cd8486dff5a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:51,177 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/B of 883bdd7db86faeb1b8a9722c1fd955c1 into 5e746d1683a7421d9116a46376ed742d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:51,177 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:51,177 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:51,177 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/A, priority=13, startTime=1731456830750; duration=0sec 2024-11-13T00:13:51,178 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/B, priority=13, startTime=1731456830750; duration=0sec 2024-11-13T00:13:51,178 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:51,178 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:13:51,178 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:A 2024-11-13T00:13:51,178 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:B 2024-11-13T00:13:51,178 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:13:51,179 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:13:51,179 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 883bdd7db86faeb1b8a9722c1fd955c1/C is initiating minor compaction (all files) 2024-11-13T00:13:51,179 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 883bdd7db86faeb1b8a9722c1fd955c1/C in TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:51,179 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/49e3fc73a6f8421c999709f21efc9d4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/c44e83e64a7c4769a1ba075a2d663df0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/b7480eedaf0349ab9a3f91c2011821b8] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp, totalSize=37.2 K 2024-11-13T00:13:51,179 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e3fc73a6f8421c999709f21efc9d4c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731456826939 2024-11-13T00:13:51,180 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44e83e64a7c4769a1ba075a2d663df0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731456827126 2024-11-13T00:13:51,180 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7480eedaf0349ab9a3f91c2011821b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731456828310 2024-11-13T00:13:51,187 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 883bdd7db86faeb1b8a9722c1fd955c1#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:13:51,187 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/3424efde0740431284d905ffb005c74f is 50, key is test_row_0/C:col10/1731456828310/Put/seqid=0 2024-11-13T00:13:51,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741951_1127 (size=13561) 2024-11-13T00:13:51,307 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/4d069eee7be8404e8a2c951625dfbe11 2024-11-13T00:13:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1bf2c6150a61427a8e0ba7ffc295c09e is 50, key is test_row_1/B:col10/1731456829528/Put/seqid=0 2024-11-13T00:13:51,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741952_1128 (size=9857) 2024-11-13T00:13:51,596 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/3424efde0740431284d905ffb005c74f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/3424efde0740431284d905ffb005c74f 2024-11-13T00:13:51,601 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883bdd7db86faeb1b8a9722c1fd955c1/C of 883bdd7db86faeb1b8a9722c1fd955c1 into 3424efde0740431284d905ffb005c74f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:13:51,601 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:51,601 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., storeName=883bdd7db86faeb1b8a9722c1fd955c1/C, priority=13, startTime=1731456830750; duration=0sec 2024-11-13T00:13:51,601 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:13:51,601 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 883bdd7db86faeb1b8a9722c1fd955c1:C 2024-11-13T00:13:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:13:51,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. as already flushing 2024-11-13T00:13:51,677 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T00:13:51,677 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:51,677 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:51,677 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:51,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:51,687 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T00:13:51,687 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:51,687 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:51,687 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:51,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:51,720 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1bf2c6150a61427a8e0ba7ffc295c09e 2024-11-13T00:13:51,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/271f43aaec6342119d24b7d3ea503f9f is 50, key is test_row_1/C:col10/1731456829528/Put/seqid=0 2024-11-13T00:13:51,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741953_1129 (size=9857) 2024-11-13T00:13:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:52,132 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/271f43aaec6342119d24b7d3ea503f9f 2024-11-13T00:13:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/4d069eee7be8404e8a2c951625dfbe11 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/4d069eee7be8404e8a2c951625dfbe11 2024-11-13T00:13:52,141 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/4d069eee7be8404e8a2c951625dfbe11, entries=100, sequenceid=534, filesize=9.6 K 2024-11-13T00:13:52,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/1bf2c6150a61427a8e0ba7ffc295c09e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1bf2c6150a61427a8e0ba7ffc295c09e 2024-11-13T00:13:52,146 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1bf2c6150a61427a8e0ba7ffc295c09e, entries=100, sequenceid=534, filesize=9.6 K 2024-11-13T00:13:52,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/271f43aaec6342119d24b7d3ea503f9f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/271f43aaec6342119d24b7d3ea503f9f 2024-11-13T00:13:52,150 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/271f43aaec6342119d24b7d3ea503f9f, entries=100, sequenceid=534, filesize=9.6 K 2024-11-13T00:13:52,151 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for 883bdd7db86faeb1b8a9722c1fd955c1 in 1253ms, sequenceid=534, compaction requested=false 2024-11-13T00:13:52,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 883bdd7db86faeb1b8a9722c1fd955c1: 2024-11-13T00:13:52,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:13:52,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-13T00:13:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-13T00:13:52,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T00:13:52,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4820 sec 2024-11-13T00:13:52,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.4870 sec 2024-11-13T00:13:52,778 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T00:13:52,778 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:13:52,778 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:52,778 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:13:52,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:13:52,822 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:13:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T00:13:53,836 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:00,328 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T00:14:00,328 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:00,328 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:00,328 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:00,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:00,418 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T00:14:00,418 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:00,418 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:00,418 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:00,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:00,418 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2639 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2710 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1201 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3594 rows 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1204 2024-11-13T00:14:00,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3603 rows 2024-11-13T00:14:00,419 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:14:00,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:14:00,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:00,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:00,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:00,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T00:14:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T00:14:00,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456840431"}]},"ts":"1731456840431"} 2024-11-13T00:14:00,434 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T00:14:00,434 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T00:14:00,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:14:00,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, UNASSIGN}] 2024-11-13T00:14:00,441 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, UNASSIGN 2024-11-13T00:14:00,443 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=883bdd7db86faeb1b8a9722c1fd955c1, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:00,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, UNASSIGN because future has completed 2024-11-13T00:14:00,445 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:14:00,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:14:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T00:14:00,603 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:14:00,603 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 883bdd7db86faeb1b8a9722c1fd955c1, disabling compactions & flushes 2024-11-13T00:14:00,604 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. after waiting 0 ms 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:14:00,604 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 883bdd7db86faeb1b8a9722c1fd955c1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=A 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=B 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 883bdd7db86faeb1b8a9722c1fd955c1, store=C 2024-11-13T00:14:00,604 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:00,610 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/103b3edfb2cc495380cd59941f103289 is 50, key is test_row_0/A:col10/1731456840417/Put/seqid=0 2024-11-13T00:14:00,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741954_1130 (size=12301) 2024-11-13T00:14:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T00:14:01,016 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/103b3edfb2cc495380cd59941f103289 2024-11-13T00:14:01,025 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/40a8432d5b5846459717e1dd93d267e8 is 50, key is test_row_0/B:col10/1731456840417/Put/seqid=0 2024-11-13T00:14:01,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741955_1131 (size=12301) 2024-11-13T00:14:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T00:14:01,430 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/40a8432d5b5846459717e1dd93d267e8 2024-11-13T00:14:01,440 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/8e18051e18034ba0987cc3e2d0e1019b is 50, key is test_row_0/C:col10/1731456840417/Put/seqid=0 2024-11-13T00:14:01,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741956_1132 (size=12301) 2024-11-13T00:14:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T00:14:01,845 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/8e18051e18034ba0987cc3e2d0e1019b 2024-11-13T00:14:01,852 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/A/103b3edfb2cc495380cd59941f103289 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/103b3edfb2cc495380cd59941f103289 2024-11-13T00:14:01,859 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/103b3edfb2cc495380cd59941f103289, entries=150, sequenceid=545, filesize=12.0 K 2024-11-13T00:14:01,860 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/B/40a8432d5b5846459717e1dd93d267e8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/40a8432d5b5846459717e1dd93d267e8 2024-11-13T00:14:01,866 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/40a8432d5b5846459717e1dd93d267e8, entries=150, sequenceid=545, filesize=12.0 K 2024-11-13T00:14:01,867 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/.tmp/C/8e18051e18034ba0987cc3e2d0e1019b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/8e18051e18034ba0987cc3e2d0e1019b 2024-11-13T00:14:01,876 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/8e18051e18034ba0987cc3e2d0e1019b, entries=150, sequenceid=545, filesize=12.0 K 2024-11-13T00:14:01,877 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 883bdd7db86faeb1b8a9722c1fd955c1 in 1273ms, sequenceid=545, compaction requested=true 2024-11-13T00:14:01,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f77190f58459493dbfd8a75f25f84526, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/dc33743e837a411b811235a6b988e3d7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d0738c4fd7f649d4a3bc80d88ed4198d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2608bff03334edc9a8eadd3d6880e41, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3f6d796699164392bb64156b1044ad97, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1040e8f5e4b444f99a300ea77d80bc2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac39ed145ba543cbbbec766b9f6b2d53, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2bf1467d9634685a3a517b3b42b8568, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fd6f83ca688c420589c2ef7e0afc212c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/9fe7924361a24cddbd91c8ea6667c597, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/59fbfe17ad024a2db2131dcf77d447d5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/20305fcb7a8d41d188944fb43bf7dbad, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2f912d600a7a4ab59c95138882195172, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/80f7b076d90a45b9b14026678e55f9c7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/70f672d2170741f995530554e4cc0df5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/afdca85527624769ac73369ebbaf4db1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/51f57b29ed5847ca9d5477401846b605, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/e6ba9921d30d4acabf39b3ef374e0a43, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0246f33320a44014840058f45803c745, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/da0e95048ec24ec38f84aee69d73ac6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/33ef34b3a83c42ec9224470cda0a7a23, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/a07e938e61234566b60351de9ca1d0a2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2975749efdd34bbd9d9d65dc95b3b11f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac4fb64b7c3040999d90c2170ce934fa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1295989b7a9435c88b9580e8799d01d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d3ab12d81ab343edb767624960d17bc1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bfa6d507649e4a9ca6471a5e19427307, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0df6c598525b4631801735b739293007, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fca4d74b711d4ddc8e4e8b6cced06440, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bc93b430f55a47c692277eb6bc5fcb51, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ce37dd22a45d46e79f3bcecda65e4fe7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/89d85c313b294cc58af8281aaf736cf6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/41f1155ac60947998fbe4c6fb74aad78, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/710ff2d973ff495486814cd2a7919a6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0fc5c81586c344dea4a47f6a4fc2cadf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/8428792b69bb46cabac6721995237659, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3c3bdbef6f614feab658605dcd85add3] to archive 2024-11-13T00:14:01,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:14:01,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f77190f58459493dbfd8a75f25f84526 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f77190f58459493dbfd8a75f25f84526 2024-11-13T00:14:01,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/dc33743e837a411b811235a6b988e3d7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/dc33743e837a411b811235a6b988e3d7 2024-11-13T00:14:01,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d0738c4fd7f649d4a3bc80d88ed4198d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d0738c4fd7f649d4a3bc80d88ed4198d 2024-11-13T00:14:01,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2608bff03334edc9a8eadd3d6880e41 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2608bff03334edc9a8eadd3d6880e41 2024-11-13T00:14:01,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3f6d796699164392bb64156b1044ad97 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3f6d796699164392bb64156b1044ad97 2024-11-13T00:14:01,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1040e8f5e4b444f99a300ea77d80bc2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1040e8f5e4b444f99a300ea77d80bc2 2024-11-13T00:14:01,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac39ed145ba543cbbbec766b9f6b2d53 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac39ed145ba543cbbbec766b9f6b2d53 2024-11-13T00:14:01,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2bf1467d9634685a3a517b3b42b8568 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/f2bf1467d9634685a3a517b3b42b8568 2024-11-13T00:14:01,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fd6f83ca688c420589c2ef7e0afc212c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fd6f83ca688c420589c2ef7e0afc212c 2024-11-13T00:14:01,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/9fe7924361a24cddbd91c8ea6667c597 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/9fe7924361a24cddbd91c8ea6667c597 2024-11-13T00:14:01,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/59fbfe17ad024a2db2131dcf77d447d5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/59fbfe17ad024a2db2131dcf77d447d5 2024-11-13T00:14:01,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/20305fcb7a8d41d188944fb43bf7dbad to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/20305fcb7a8d41d188944fb43bf7dbad 2024-11-13T00:14:01,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2f912d600a7a4ab59c95138882195172 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2f912d600a7a4ab59c95138882195172 2024-11-13T00:14:01,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/80f7b076d90a45b9b14026678e55f9c7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/80f7b076d90a45b9b14026678e55f9c7 2024-11-13T00:14:01,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/70f672d2170741f995530554e4cc0df5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/70f672d2170741f995530554e4cc0df5 2024-11-13T00:14:01,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/afdca85527624769ac73369ebbaf4db1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/afdca85527624769ac73369ebbaf4db1 2024-11-13T00:14:01,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/51f57b29ed5847ca9d5477401846b605 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/51f57b29ed5847ca9d5477401846b605 2024-11-13T00:14:01,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/e6ba9921d30d4acabf39b3ef374e0a43 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/e6ba9921d30d4acabf39b3ef374e0a43 2024-11-13T00:14:01,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0246f33320a44014840058f45803c745 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0246f33320a44014840058f45803c745 2024-11-13T00:14:01,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/da0e95048ec24ec38f84aee69d73ac6d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/da0e95048ec24ec38f84aee69d73ac6d 2024-11-13T00:14:01,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/33ef34b3a83c42ec9224470cda0a7a23 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/33ef34b3a83c42ec9224470cda0a7a23 2024-11-13T00:14:01,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/a07e938e61234566b60351de9ca1d0a2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/a07e938e61234566b60351de9ca1d0a2 2024-11-13T00:14:01,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2975749efdd34bbd9d9d65dc95b3b11f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/2975749efdd34bbd9d9d65dc95b3b11f 2024-11-13T00:14:01,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac4fb64b7c3040999d90c2170ce934fa to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ac4fb64b7c3040999d90c2170ce934fa 2024-11-13T00:14:01,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1295989b7a9435c88b9580e8799d01d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d1295989b7a9435c88b9580e8799d01d 2024-11-13T00:14:01,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d3ab12d81ab343edb767624960d17bc1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/d3ab12d81ab343edb767624960d17bc1 2024-11-13T00:14:01,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bfa6d507649e4a9ca6471a5e19427307 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bfa6d507649e4a9ca6471a5e19427307 2024-11-13T00:14:01,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0df6c598525b4631801735b739293007 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0df6c598525b4631801735b739293007 2024-11-13T00:14:01,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fca4d74b711d4ddc8e4e8b6cced06440 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/fca4d74b711d4ddc8e4e8b6cced06440 2024-11-13T00:14:01,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bc93b430f55a47c692277eb6bc5fcb51 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/bc93b430f55a47c692277eb6bc5fcb51 2024-11-13T00:14:01,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ce37dd22a45d46e79f3bcecda65e4fe7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/ce37dd22a45d46e79f3bcecda65e4fe7 2024-11-13T00:14:01,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/89d85c313b294cc58af8281aaf736cf6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/89d85c313b294cc58af8281aaf736cf6 2024-11-13T00:14:01,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/41f1155ac60947998fbe4c6fb74aad78 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/41f1155ac60947998fbe4c6fb74aad78 2024-11-13T00:14:01,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/710ff2d973ff495486814cd2a7919a6d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/710ff2d973ff495486814cd2a7919a6d 2024-11-13T00:14:01,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0fc5c81586c344dea4a47f6a4fc2cadf to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/0fc5c81586c344dea4a47f6a4fc2cadf 2024-11-13T00:14:01,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/8428792b69bb46cabac6721995237659 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/8428792b69bb46cabac6721995237659 2024-11-13T00:14:01,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3c3bdbef6f614feab658605dcd85add3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3c3bdbef6f614feab658605dcd85add3 2024-11-13T00:14:01,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9fa32bfaccac47438ae2ff3a44d2a1a4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6705c516a53a457c84bab76ca8b5911e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b83c1d665145477ea540efbe03beaa4d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/ed941b1aa8e14ee39b234baac656961f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/7e23cd94512b4f6ea7e0af99d1a9b864, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e376a83328ea4cbd8fce9bf1fea3cac8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1b7ea6e9d5934bb68b69b0cce1616dfa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0ea6369e72ed49a59e98e99874a4651e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/fb8fde2edd944bf39e2f249bf48987d0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0cfcdb6503194555919768b059d395da, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6e90f3e2ee9243acb7bcbecd728df3fb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/739c921be5d24585a2828e1c0c99a0cc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/a95a7a6ed8e04df7ad97db3305137e56, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/f964af3e8373415ab06e6df456bb6fc9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/494934b6ad084302b5e87d5fc75d6197, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/c58753577fb14412b8e767e51bdea617, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/bfe7b43f58b24d4196000dc3724cef61, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/041d536b71904ab684d0b7fdca483fce, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e82b6f11548349f0a599ba5551073019, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/073bb664c84b42a6b20edcfd66bcae18, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/79915a300ddd4d898480da19fc2d2345, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/57031257478843afb8da6690bc61dd1f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1fae5993d6464a61abbb3a30030e34c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/eb58fb61894340cf8640fd02ac09668d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/421c3f62553a4a5ba6fb5d81573775e4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db265ade19324f7f86bced5a1cb99ced, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b441a0237a994150b1b7c9b1d10d6b74, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/3257e78cd84f4f9a82854e990866921f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/4a27a3014cd64f3383bcd720eb64b883, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9d2bec4da8f14f68a67746c8e052cbf2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/8e29adfcfeb7457498e98f106cec5d2d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5f28db9eba0046168fa3be76c95d81fd, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/87b2cc6e3f4d47da9975646514434433, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9ae5188d0dea4902a84b15e55919c043, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/22ea575447084db481492870e513f65b] to archive 2024-11-13T00:14:01,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:14:01,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9fa32bfaccac47438ae2ff3a44d2a1a4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9fa32bfaccac47438ae2ff3a44d2a1a4 2024-11-13T00:14:01,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6705c516a53a457c84bab76ca8b5911e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6705c516a53a457c84bab76ca8b5911e 2024-11-13T00:14:01,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b83c1d665145477ea540efbe03beaa4d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b83c1d665145477ea540efbe03beaa4d 2024-11-13T00:14:01,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/ed941b1aa8e14ee39b234baac656961f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/ed941b1aa8e14ee39b234baac656961f 2024-11-13T00:14:01,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/7e23cd94512b4f6ea7e0af99d1a9b864 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/7e23cd94512b4f6ea7e0af99d1a9b864 2024-11-13T00:14:01,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e376a83328ea4cbd8fce9bf1fea3cac8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e376a83328ea4cbd8fce9bf1fea3cac8 2024-11-13T00:14:01,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1b7ea6e9d5934bb68b69b0cce1616dfa to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1b7ea6e9d5934bb68b69b0cce1616dfa 2024-11-13T00:14:01,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0ea6369e72ed49a59e98e99874a4651e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0ea6369e72ed49a59e98e99874a4651e 2024-11-13T00:14:01,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/fb8fde2edd944bf39e2f249bf48987d0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/fb8fde2edd944bf39e2f249bf48987d0 2024-11-13T00:14:01,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0cfcdb6503194555919768b059d395da to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/0cfcdb6503194555919768b059d395da 2024-11-13T00:14:01,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6e90f3e2ee9243acb7bcbecd728df3fb to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/6e90f3e2ee9243acb7bcbecd728df3fb 2024-11-13T00:14:01,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/739c921be5d24585a2828e1c0c99a0cc to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/739c921be5d24585a2828e1c0c99a0cc 2024-11-13T00:14:01,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/a95a7a6ed8e04df7ad97db3305137e56 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/a95a7a6ed8e04df7ad97db3305137e56 2024-11-13T00:14:01,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/f964af3e8373415ab06e6df456bb6fc9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/f964af3e8373415ab06e6df456bb6fc9 2024-11-13T00:14:01,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/494934b6ad084302b5e87d5fc75d6197 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/494934b6ad084302b5e87d5fc75d6197 2024-11-13T00:14:01,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/12c3fbda8c4c40cc9bbf8deecb5c6f0d 2024-11-13T00:14:01,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/c58753577fb14412b8e767e51bdea617 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/c58753577fb14412b8e767e51bdea617 2024-11-13T00:14:01,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/bfe7b43f58b24d4196000dc3724cef61 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/bfe7b43f58b24d4196000dc3724cef61 2024-11-13T00:14:01,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/041d536b71904ab684d0b7fdca483fce to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/041d536b71904ab684d0b7fdca483fce 2024-11-13T00:14:01,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e82b6f11548349f0a599ba5551073019 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/e82b6f11548349f0a599ba5551073019 2024-11-13T00:14:01,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db5cdeb4c3d84ec48ba9265d8e9ac5d0 2024-11-13T00:14:01,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/073bb664c84b42a6b20edcfd66bcae18 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/073bb664c84b42a6b20edcfd66bcae18 2024-11-13T00:14:02,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/79915a300ddd4d898480da19fc2d2345 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/79915a300ddd4d898480da19fc2d2345 2024-11-13T00:14:02,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/57031257478843afb8da6690bc61dd1f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/57031257478843afb8da6690bc61dd1f 2024-11-13T00:14:02,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1fae5993d6464a61abbb3a30030e34c6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1fae5993d6464a61abbb3a30030e34c6 2024-11-13T00:14:02,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/eb58fb61894340cf8640fd02ac09668d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/eb58fb61894340cf8640fd02ac09668d 2024-11-13T00:14:02,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/421c3f62553a4a5ba6fb5d81573775e4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/421c3f62553a4a5ba6fb5d81573775e4 2024-11-13T00:14:02,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db265ade19324f7f86bced5a1cb99ced to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/db265ade19324f7f86bced5a1cb99ced 2024-11-13T00:14:02,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b441a0237a994150b1b7c9b1d10d6b74 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/b441a0237a994150b1b7c9b1d10d6b74 2024-11-13T00:14:02,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/3257e78cd84f4f9a82854e990866921f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/3257e78cd84f4f9a82854e990866921f 2024-11-13T00:14:02,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/4a27a3014cd64f3383bcd720eb64b883 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/4a27a3014cd64f3383bcd720eb64b883 2024-11-13T00:14:02,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9d2bec4da8f14f68a67746c8e052cbf2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9d2bec4da8f14f68a67746c8e052cbf2 2024-11-13T00:14:02,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/8e29adfcfeb7457498e98f106cec5d2d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/8e29adfcfeb7457498e98f106cec5d2d 2024-11-13T00:14:02,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5f28db9eba0046168fa3be76c95d81fd to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5f28db9eba0046168fa3be76c95d81fd 2024-11-13T00:14:02,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/87b2cc6e3f4d47da9975646514434433 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/87b2cc6e3f4d47da9975646514434433 2024-11-13T00:14:02,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9ae5188d0dea4902a84b15e55919c043 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/9ae5188d0dea4902a84b15e55919c043 2024-11-13T00:14:02,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/22ea575447084db481492870e513f65b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/22ea575447084db481492870e513f65b 2024-11-13T00:14:02,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/1e821e16141e410ab2e566212519ec7d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/06973b619b3f44e5bcad8cfb5a70f496, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d1e11dc465d14288a9d95e9423b73f1a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/db4a7f354e8343cda2872ed26f62fb99, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/a2cfb863dde643cd8f898aff6b7acc19, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/41f08dfbc431452291e2f37a9089085f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/032470e9cca54154a2ecb3841d447ae7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/872babdd735e47769cdfa3d35a684c4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d782bc47a7074628be8e2a3e3a393d69, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/503aba9d09be4d3f861bb247ca73ffd5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/52630e3c27d845318de1abb379661b59, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/08aef294a9fa4621b351f2218bcebc95, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/e0d8e56ce5984aaaac13f2f1475eb94a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/83ea5cff08524971b771372d97a784be, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/aea029f0ccaa40a797f0e875ff2175b5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34205403a26e4fd891f2d244582f1822, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/fd1989f118fa4da8b9e61c81af4dfd32, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/17838e8111b042e1b5db9c946102b613, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/ff3bedd030f84d2eb05ace3a3503d5a9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/334fd291baa540a6aeb6cc4eeb2b8bb2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/25b4767aab9b4257beb388e6ebd27f17, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34865e59f9264ea79740f5d2e98802e7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9a08a61fd11f47ab9dcf4a1cd37ab367, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/4ceaafd18773417fa69aca26dae31f5e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9b56cd96ce4c489a9e12876ba5cda08c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/6acf4908ecac45b0ae9208c142cf536a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/2c91dd2f294647afb11942d5241ed55d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/51b2a0ee95a645529e24422d3e34d910, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/935ff3c17d0242c1b6a4a420867430e3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/01b8c569e19c4b4c84f8769ebaea5b74, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/bea2699403414abb95ede1d40c3d60a3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7c7c4d84868b47b2b77cc1c4af0ce780, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/49e3fc73a6f8421c999709f21efc9d4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/5fb487575aa34cc8bb11949b58cc7baf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/c44e83e64a7c4769a1ba075a2d663df0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/b7480eedaf0349ab9a3f91c2011821b8] to archive 2024-11-13T00:14:02,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:14:02,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/1e821e16141e410ab2e566212519ec7d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/1e821e16141e410ab2e566212519ec7d 2024-11-13T00:14:02,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/06973b619b3f44e5bcad8cfb5a70f496 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/06973b619b3f44e5bcad8cfb5a70f496 2024-11-13T00:14:02,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d1e11dc465d14288a9d95e9423b73f1a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d1e11dc465d14288a9d95e9423b73f1a 2024-11-13T00:14:02,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/db4a7f354e8343cda2872ed26f62fb99 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/db4a7f354e8343cda2872ed26f62fb99 2024-11-13T00:14:02,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/a2cfb863dde643cd8f898aff6b7acc19 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/a2cfb863dde643cd8f898aff6b7acc19 2024-11-13T00:14:02,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/41f08dfbc431452291e2f37a9089085f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/41f08dfbc431452291e2f37a9089085f 2024-11-13T00:14:02,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/032470e9cca54154a2ecb3841d447ae7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/032470e9cca54154a2ecb3841d447ae7 2024-11-13T00:14:02,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/872babdd735e47769cdfa3d35a684c4c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/872babdd735e47769cdfa3d35a684c4c 2024-11-13T00:14:02,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d782bc47a7074628be8e2a3e3a393d69 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/d782bc47a7074628be8e2a3e3a393d69 2024-11-13T00:14:02,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/503aba9d09be4d3f861bb247ca73ffd5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/503aba9d09be4d3f861bb247ca73ffd5 2024-11-13T00:14:02,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/52630e3c27d845318de1abb379661b59 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/52630e3c27d845318de1abb379661b59 2024-11-13T00:14:02,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/08aef294a9fa4621b351f2218bcebc95 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/08aef294a9fa4621b351f2218bcebc95 2024-11-13T00:14:02,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/e0d8e56ce5984aaaac13f2f1475eb94a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/e0d8e56ce5984aaaac13f2f1475eb94a 2024-11-13T00:14:02,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/83ea5cff08524971b771372d97a784be to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/83ea5cff08524971b771372d97a784be 2024-11-13T00:14:02,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/aea029f0ccaa40a797f0e875ff2175b5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/aea029f0ccaa40a797f0e875ff2175b5 2024-11-13T00:14:02,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34205403a26e4fd891f2d244582f1822 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34205403a26e4fd891f2d244582f1822 2024-11-13T00:14:02,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/fd1989f118fa4da8b9e61c81af4dfd32 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/fd1989f118fa4da8b9e61c81af4dfd32 2024-11-13T00:14:02,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/17838e8111b042e1b5db9c946102b613 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/17838e8111b042e1b5db9c946102b613 2024-11-13T00:14:02,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/ff3bedd030f84d2eb05ace3a3503d5a9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/ff3bedd030f84d2eb05ace3a3503d5a9 2024-11-13T00:14:02,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/334fd291baa540a6aeb6cc4eeb2b8bb2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/334fd291baa540a6aeb6cc4eeb2b8bb2 2024-11-13T00:14:02,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7a0bbaa1ce124f8ea3c4ffbe5125f66d 2024-11-13T00:14:02,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/25b4767aab9b4257beb388e6ebd27f17 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/25b4767aab9b4257beb388e6ebd27f17 2024-11-13T00:14:02,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34865e59f9264ea79740f5d2e98802e7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/34865e59f9264ea79740f5d2e98802e7 2024-11-13T00:14:02,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9a08a61fd11f47ab9dcf4a1cd37ab367 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9a08a61fd11f47ab9dcf4a1cd37ab367 2024-11-13T00:14:02,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/4ceaafd18773417fa69aca26dae31f5e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/4ceaafd18773417fa69aca26dae31f5e 2024-11-13T00:14:02,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9b56cd96ce4c489a9e12876ba5cda08c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/9b56cd96ce4c489a9e12876ba5cda08c 2024-11-13T00:14:02,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/6acf4908ecac45b0ae9208c142cf536a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/6acf4908ecac45b0ae9208c142cf536a 2024-11-13T00:14:02,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/2c91dd2f294647afb11942d5241ed55d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/2c91dd2f294647afb11942d5241ed55d 2024-11-13T00:14:02,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/51b2a0ee95a645529e24422d3e34d910 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/51b2a0ee95a645529e24422d3e34d910 2024-11-13T00:14:02,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/935ff3c17d0242c1b6a4a420867430e3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/935ff3c17d0242c1b6a4a420867430e3 2024-11-13T00:14:02,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/01b8c569e19c4b4c84f8769ebaea5b74 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/01b8c569e19c4b4c84f8769ebaea5b74 2024-11-13T00:14:02,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/bea2699403414abb95ede1d40c3d60a3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/bea2699403414abb95ede1d40c3d60a3 2024-11-13T00:14:02,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7c7c4d84868b47b2b77cc1c4af0ce780 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/7c7c4d84868b47b2b77cc1c4af0ce780 2024-11-13T00:14:02,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/49e3fc73a6f8421c999709f21efc9d4c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/49e3fc73a6f8421c999709f21efc9d4c 2024-11-13T00:14:02,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/5fb487575aa34cc8bb11949b58cc7baf to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/5fb487575aa34cc8bb11949b58cc7baf 2024-11-13T00:14:02,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/c44e83e64a7c4769a1ba075a2d663df0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/c44e83e64a7c4769a1ba075a2d663df0 2024-11-13T00:14:02,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/b7480eedaf0349ab9a3f91c2011821b8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/b7480eedaf0349ab9a3f91c2011821b8 2024-11-13T00:14:02,093 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/recovered.edits/548.seqid, newMaxSeqId=548, maxSeqId=1 2024-11-13T00:14:02,094 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. 2024-11-13T00:14:02,095 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 883bdd7db86faeb1b8a9722c1fd955c1: Waiting for close lock at 1731456840603Running coprocessor pre-close hooks at 1731456840603Disabling compacts and flushes for region at 1731456840603Disabling writes for close at 1731456840604 (+1 ms)Obtaining lock to block concurrent updates at 1731456840604Preparing flush snapshotting stores in 883bdd7db86faeb1b8a9722c1fd955c1 at 1731456840604Finished memstore snapshotting TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731456840604Flushing stores of TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1. at 1731456840605 (+1 ms)Flushing 883bdd7db86faeb1b8a9722c1fd955c1/A: creating writer at 1731456840605Flushing 883bdd7db86faeb1b8a9722c1fd955c1/A: appending metadata at 1731456840610 (+5 ms)Flushing 883bdd7db86faeb1b8a9722c1fd955c1/A: closing flushed file at 1731456840610Flushing 883bdd7db86faeb1b8a9722c1fd955c1/B: creating writer at 1731456841021 (+411 ms)Flushing 883bdd7db86faeb1b8a9722c1fd955c1/B: appending metadata at 1731456841024 (+3 ms)Flushing 883bdd7db86faeb1b8a9722c1fd955c1/B: closing flushed file at 1731456841024Flushing 883bdd7db86faeb1b8a9722c1fd955c1/C: creating writer at 1731456841436 (+412 ms)Flushing 883bdd7db86faeb1b8a9722c1fd955c1/C: appending metadata at 1731456841439 (+3 ms)Flushing 883bdd7db86faeb1b8a9722c1fd955c1/C: closing flushed file at 1731456841439Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@141d4062: reopening flushed file at 1731456841851 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1022e111: reopening flushed file at 1731456841859 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ced1dc: reopening flushed file at 1731456841866 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 883bdd7db86faeb1b8a9722c1fd955c1 in 1273ms, sequenceid=545, compaction requested=true at 1731456841877 (+11 ms)Writing region close event to WAL at 1731456842084 (+207 ms)Running coprocessor post-close hooks at 1731456842094 (+10 ms)Closed at 1731456842094 2024-11-13T00:14:02,099 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=883bdd7db86faeb1b8a9722c1fd955c1, regionState=CLOSED 2024-11-13T00:14:02,099 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:14:02,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:14:02,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-13T00:14:02,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 883bdd7db86faeb1b8a9722c1fd955c1, server=cb28580903fb,45273,1731456804594 in 1.6560 sec 2024-11-13T00:14:02,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T00:14:02,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=883bdd7db86faeb1b8a9722c1fd955c1, UNASSIGN in 1.6640 sec 2024-11-13T00:14:02,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-13T00:14:02,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6710 sec 2024-11-13T00:14:02,109 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456842109"}]},"ts":"1731456842109"} 2024-11-13T00:14:02,110 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T00:14:02,111 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T00:14:02,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6860 sec 2024-11-13T00:14:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T00:14:02,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:02,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T00:14:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,586 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-13T00:14:02,589 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,595 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:14:02,600 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/recovered.edits] 2024-11-13T00:14:02,604 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/103b3edfb2cc495380cd59941f103289 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/103b3edfb2cc495380cd59941f103289 2024-11-13T00:14:02,606 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3fba868b02be48ef99622cd8486dff5a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/3fba868b02be48ef99622cd8486dff5a 2024-11-13T00:14:02,607 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/4d069eee7be8404e8a2c951625dfbe11 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/A/4d069eee7be8404e8a2c951625dfbe11 2024-11-13T00:14:02,611 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1bf2c6150a61427a8e0ba7ffc295c09e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/1bf2c6150a61427a8e0ba7ffc295c09e 2024-11-13T00:14:02,613 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/40a8432d5b5846459717e1dd93d267e8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/40a8432d5b5846459717e1dd93d267e8 2024-11-13T00:14:02,614 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5e746d1683a7421d9116a46376ed742d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/B/5e746d1683a7421d9116a46376ed742d 2024-11-13T00:14:02,617 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/271f43aaec6342119d24b7d3ea503f9f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/271f43aaec6342119d24b7d3ea503f9f 2024-11-13T00:14:02,619 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/3424efde0740431284d905ffb005c74f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/3424efde0740431284d905ffb005c74f 2024-11-13T00:14:02,620 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/8e18051e18034ba0987cc3e2d0e1019b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/C/8e18051e18034ba0987cc3e2d0e1019b 2024-11-13T00:14:02,624 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/recovered.edits/548.seqid to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1/recovered.edits/548.seqid 2024-11-13T00:14:02,624 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/883bdd7db86faeb1b8a9722c1fd955c1 2024-11-13T00:14:02,625 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T00:14:02,630 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T00:14:02,640 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T00:14:02,680 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T00:14:02,682 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,683 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T00:14:02,683 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731456842683"}]},"ts":"9223372036854775807"} 2024-11-13T00:14:02,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T00:14:02,689 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 883bdd7db86faeb1b8a9722c1fd955c1, NAME => 'TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T00:14:02,689 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T00:14:02,690 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731456842690"}]},"ts":"9223372036854775807"} 2024-11-13T00:14:02,694 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T00:14:02,696 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-13T00:14:02,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 115 msec 2024-11-13T00:14:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-13T00:14:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T00:14:02,907 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:02,920 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1970940896_22 at /127.0.0.1:58364 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/cb28580903fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;cb28580903fb:45273-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 314) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2077 (was 2124) 2024-11-13T00:14:02,931 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=2076 2024-11-13T00:14:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:14:02,934 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:14:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:02,936 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:14:02,936 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:02,936 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-13T00:14:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T00:14:02,937 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:14:02,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741957_1133 (size=1061) 2024-11-13T00:14:02,948 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:14:02,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741958_1134 (size=53) 2024-11-13T00:14:02,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:14:02,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1ee97d494479a45762cab6f40ed6001d, disabling compactions & flushes 2024-11-13T00:14:02,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:02,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:02,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. after waiting 0 ms 2024-11-13T00:14:02,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:02,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:02,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ee97d494479a45762cab6f40ed6001d: Waiting for close lock at 1731456842959Disabling compacts and flushes for region at 1731456842959Disabling writes for close at 1731456842959Writing region close event to WAL at 1731456842959Closed at 1731456842959 2024-11-13T00:14:02,961 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:14:02,962 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731456842961"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731456842961"}]},"ts":"1731456842961"} 2024-11-13T00:14:02,964 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:14:02,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:14:02,965 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456842965"}]},"ts":"1731456842965"} 2024-11-13T00:14:02,967 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T00:14:02,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, ASSIGN}] 2024-11-13T00:14:02,968 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, ASSIGN 2024-11-13T00:14:02,969 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, ASSIGN; state=OFFLINE, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=false 2024-11-13T00:14:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T00:14:03,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:03,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, ASSIGN because future has completed 2024-11-13T00:14:03,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:14:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T00:14:03,281 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:03,282 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:14:03,282 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,282 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:14:03,282 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,282 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,284 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,286 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:03,286 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ee97d494479a45762cab6f40ed6001d columnFamilyName A 2024-11-13T00:14:03,286 DEBUG [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:03,287 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(327): Store=1ee97d494479a45762cab6f40ed6001d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:03,287 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,288 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:03,289 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ee97d494479a45762cab6f40ed6001d columnFamilyName B 2024-11-13T00:14:03,289 DEBUG [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:03,289 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(327): Store=1ee97d494479a45762cab6f40ed6001d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:03,289 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,290 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:03,290 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ee97d494479a45762cab6f40ed6001d columnFamilyName C 2024-11-13T00:14:03,290 DEBUG [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:03,291 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(327): Store=1ee97d494479a45762cab6f40ed6001d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:03,291 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:03,292 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,292 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,293 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,294 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,294 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,294 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:14:03,296 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,298 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:14:03,298 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 1ee97d494479a45762cab6f40ed6001d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64267737, jitterRate=-0.04233609139919281}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:14:03,298 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:03,299 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 1ee97d494479a45762cab6f40ed6001d: Running coprocessor pre-open hook at 1731456843283Writing region info on filesystem at 1731456843283Initializing all the Stores at 1731456843284 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456843284Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456843284Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456843284Cleaning up temporary data from old regions at 1731456843294 (+10 ms)Running coprocessor post-open hooks at 1731456843298 (+4 ms)Region opened successfully at 1731456843299 (+1 ms) 2024-11-13T00:14:03,300 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., pid=38, masterSystemTime=1731456843277 2024-11-13T00:14:03,302 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:03,302 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:03,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:03,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:14:03,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-13T00:14:03,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 in 182 msec 2024-11-13T00:14:03,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T00:14:03,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, ASSIGN in 342 msec 2024-11-13T00:14:03,313 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:14:03,313 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456843313"}]},"ts":"1731456843313"} 2024-11-13T00:14:03,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T00:14:03,315 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:14:03,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 382 msec 2024-11-13T00:14:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T00:14:03,566 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:03,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328db2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:03,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:03,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:03,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:03,571 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:03,572 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:03,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:03,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-13T00:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:14:03,578 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:14:03,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:14:03,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:14:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:03,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741959_1135 (size=1097) 2024-11-13T00:14:04,008 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T00:14:04,009 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T00:14:04,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:14:04,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, REOPEN/MOVE}] 2024-11-13T00:14:04,021 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, REOPEN/MOVE 2024-11-13T00:14:04,022 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, REOPEN/MOVE because future has completed 2024-11-13T00:14:04,026 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:14:04,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:14:04,180 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,180 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:14:04,180 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 1ee97d494479a45762cab6f40ed6001d, disabling compactions & flushes 2024-11-13T00:14:04,180 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,180 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,180 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. after waiting 0 ms 2024-11-13T00:14:04,180 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,185 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T00:14:04,186 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,186 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 1ee97d494479a45762cab6f40ed6001d: Waiting for close lock at 1731456844180Running coprocessor pre-close hooks at 1731456844180Disabling compacts and flushes for region at 1731456844180Disabling writes for close at 1731456844180Writing region close event to WAL at 1731456844180Running coprocessor post-close hooks at 1731456844186 (+6 ms)Closed at 1731456844186 2024-11-13T00:14:04,186 WARN [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 1ee97d494479a45762cab6f40ed6001d to self. 2024-11-13T00:14:04,190 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,191 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=CLOSED 2024-11-13T00:14:04,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:14:04,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-13T00:14:04,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 in 168 msec 2024-11-13T00:14:04,206 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, REOPEN/MOVE; state=CLOSED, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=true 2024-11-13T00:14:04,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T00:14:04,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, REOPEN/MOVE because future has completed 2024-11-13T00:14:04,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:14:04,516 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,516 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:14:04,517 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,517 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:14:04,517 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,517 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,523 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,524 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:04,531 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ee97d494479a45762cab6f40ed6001d columnFamilyName A 2024-11-13T00:14:04,533 DEBUG [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:04,534 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(327): Store=1ee97d494479a45762cab6f40ed6001d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:04,535 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,536 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:04,536 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ee97d494479a45762cab6f40ed6001d columnFamilyName B 2024-11-13T00:14:04,536 DEBUG [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:04,537 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(327): Store=1ee97d494479a45762cab6f40ed6001d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:04,537 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,538 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:04,538 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ee97d494479a45762cab6f40ed6001d columnFamilyName C 2024-11-13T00:14:04,538 DEBUG [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:04,541 INFO [StoreOpener-1ee97d494479a45762cab6f40ed6001d-1 {}] regionserver.HStore(327): Store=1ee97d494479a45762cab6f40ed6001d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:04,541 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,541 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,542 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,544 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,546 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,546 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,546 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:14:04,552 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,553 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 1ee97d494479a45762cab6f40ed6001d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63004257, jitterRate=-0.06116341054439545}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:14:04,553 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,554 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 1ee97d494479a45762cab6f40ed6001d: Running coprocessor pre-open hook at 1731456844517Writing region info on filesystem at 1731456844517Initializing all the Stores at 1731456844518 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456844518Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456844519 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456844519Cleaning up temporary data from old regions at 1731456844546 (+27 ms)Running coprocessor post-open hooks at 1731456844553 (+7 ms)Region opened successfully at 1731456844554 (+1 ms) 2024-11-13T00:14:04,555 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., pid=43, masterSystemTime=1731456844511 2024-11-13T00:14:04,558 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=OPEN, openSeqNum=5, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,558 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,559 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:14:04,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-13T00:14:04,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 in 202 msec 2024-11-13T00:14:04,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T00:14:04,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, REOPEN/MOVE in 543 msec 2024-11-13T00:14:04,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-13T00:14:04,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 556 msec 2024-11-13T00:14:04,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 989 msec 2024-11-13T00:14:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-13T00:14:04,580 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:04,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe200e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,585 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d113fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3166f8b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,596 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53356dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,598 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e110111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,600 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f76cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd424e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,614 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a85fdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,618 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8806bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1040cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed9cd7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:04,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:04,640 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1ab42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:04,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:04,649 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,661 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,673 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:04,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:14:04,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:04,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:04,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:04,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:04,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:04,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T00:14:04,687 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:04,688 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:04,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:04,717 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456844707, max timestamp=1731456844716Num uniques 859; 2024-11-13T00:14:04,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456844707, max timestamp=1731456844716Num uniques 859; 2024-11-13T00:14:04,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:04,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456844707, max timestamp=1731456844716Num uniques 859; 2024-11-13T00:14:04,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:04,733 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,746 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,753 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,761 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,768 DEBUG [Thread-677 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,768 DEBUG [Thread-677 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,768 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:04,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:04,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308f70177cd984b02a4f0b19859181ba0_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_1/A:col10/1731456844672/Put/seqid=0 2024-11-13T00:14:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456904768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:14:04,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456904784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456904786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456904787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456904788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:04,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741960_1136 (size=9714) 2024-11-13T00:14:04,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:04,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456904897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456904897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456904906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456904906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456904906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:04,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:04,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:04,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:05,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456905119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456905128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456905128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456905128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456905136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:05,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308f70177cd984b02a4f0b19859181ba0_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308f70177cd984b02a4f0b19859181ba0_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:05,244 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1011b8f222074026b24a53293cd5659e, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:05,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1011b8f222074026b24a53293cd5659e is 175, key is test_row_1/A:col10/1731456844672/Put/seqid=0 2024-11-13T00:14:05,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741961_1137 (size=22361) 2024-11-13T00:14:05,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1011b8f222074026b24a53293cd5659e 2024-11-13T00:14:05,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:05,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:05,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a10fd20937f84303b0565604934796fe is 50, key is test_row_1/B:col10/1731456844672/Put/seqid=0 2024-11-13T00:14:05,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741962_1138 (size=9657) 2024-11-13T00:14:05,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a10fd20937f84303b0565604934796fe 2024-11-13T00:14:05,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/4d58213203244a7dba40af964674ae97 is 50, key is test_row_1/C:col10/1731456844672/Put/seqid=0 2024-11-13T00:14:05,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741963_1139 (size=9657) 2024-11-13T00:14:05,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/4d58213203244a7dba40af964674ae97 2024-11-13T00:14:05,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1011b8f222074026b24a53293cd5659e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e 2024-11-13T00:14:05,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e, entries=100, sequenceid=14, filesize=21.8 K 2024-11-13T00:14:05,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a10fd20937f84303b0565604934796fe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a10fd20937f84303b0565604934796fe 2024-11-13T00:14:05,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a10fd20937f84303b0565604934796fe, entries=100, sequenceid=14, filesize=9.4 K 2024-11-13T00:14:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/4d58213203244a7dba40af964674ae97 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d58213203244a7dba40af964674ae97 2024-11-13T00:14:05,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d58213203244a7dba40af964674ae97, entries=100, sequenceid=14, filesize=9.4 K 2024-11-13T00:14:05,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1ee97d494479a45762cab6f40ed6001d in 760ms, sequenceid=14, compaction requested=false 2024-11-13T00:14:05,434 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T00:14:05,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:05,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-13T00:14:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:05,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:05,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:05,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456905464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456905470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456905473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456905473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456905475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a286dad0640b425c98ea1dc63c0343db_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456845455/Put/seqid=0 2024-11-13T00:14:05,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741964_1140 (size=12154) 2024-11-13T00:14:05,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:05,557 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a286dad0640b425c98ea1dc63c0343db_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a286dad0640b425c98ea1dc63c0343db_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:05,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/96f368467f9f47a8a6731d6bfc1b69b6, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:05,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/96f368467f9f47a8a6731d6bfc1b69b6 is 175, key is test_row_0/A:col10/1731456845455/Put/seqid=0 2024-11-13T00:14:05,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456905576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456905586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741965_1141 (size=30955) 2024-11-13T00:14:05,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/96f368467f9f47a8a6731d6bfc1b69b6 2024-11-13T00:14:05,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456905596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456905596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456905596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5a2eca25d9014852b3055b3211633014 is 50, key is test_row_0/B:col10/1731456845455/Put/seqid=0 2024-11-13T00:14:05,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:05,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:05,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741966_1142 (size=12001) 2024-11-13T00:14:05,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456905786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456905796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456905806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456905806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456905806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:05,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:05,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5a2eca25d9014852b3055b3211633014 2024-11-13T00:14:06,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/cc6007d2909046ecba721dff65a0bb67 is 50, key is test_row_0/C:col10/1731456845455/Put/seqid=0 2024-11-13T00:14:06,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741967_1143 (size=12001) 2024-11-13T00:14:06,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/cc6007d2909046ecba721dff65a0bb67 2024-11-13T00:14:06,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456906096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/96f368467f9f47a8a6731d6bfc1b69b6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6 2024-11-13T00:14:06,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456906106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6, entries=150, sequenceid=52, filesize=30.2 K 2024-11-13T00:14:06,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5a2eca25d9014852b3055b3211633014 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5a2eca25d9014852b3055b3211633014 2024-11-13T00:14:06,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5a2eca25d9014852b3055b3211633014, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T00:14:06,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/cc6007d2909046ecba721dff65a0bb67 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cc6007d2909046ecba721dff65a0bb67 2024-11-13T00:14:06,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456906126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cc6007d2909046ecba721dff65a0bb67, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T00:14:06,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 1ee97d494479a45762cab6f40ed6001d in 677ms, sequenceid=52, compaction requested=false 2024-11-13T00:14:06,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:06,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:14:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:06,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fd68544d3f1d4c7280ed7a50c0391548_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741968_1144 (size=12154) 2024-11-13T00:14:06,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:06,207 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fd68544d3f1d4c7280ed7a50c0391548_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fd68544d3f1d4c7280ed7a50c0391548_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:06,208 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/52fecda1679b400ba5bb907ed859241b, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:06,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/52fecda1679b400ba5bb907ed859241b is 175, key is test_row_0/A:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741969_1145 (size=30955) 2024-11-13T00:14:06,219 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/52fecda1679b400ba5bb907ed859241b 2024-11-13T00:14:06,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/bc762089319648a68f38a657238d6f51 is 50, key is test_row_0/B:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456846203, max timestamp=1731456846262Num uniques 859; 2024-11-13T00:14:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731456846203, max timestamp=1731456846262Num uniques 859; 2024-11-13T00:14:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:14:06,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T00:14:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741970_1146 (size=12001) 2024-11-13T00:14:06,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/bc762089319648a68f38a657238d6f51 2024-11-13T00:14:06,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456906326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/91ea12042c324ebca6a0dd2a963d81aa is 50, key is test_row_0/C:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456906327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741971_1147 (size=12001) 2024-11-13T00:14:06,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/91ea12042c324ebca6a0dd2a963d81aa 2024-11-13T00:14:06,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/52fecda1679b400ba5bb907ed859241b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b 2024-11-13T00:14:06,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b, entries=150, sequenceid=63, filesize=30.2 K 2024-11-13T00:14:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/bc762089319648a68f38a657238d6f51 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bc762089319648a68f38a657238d6f51 2024-11-13T00:14:06,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bc762089319648a68f38a657238d6f51, entries=150, sequenceid=63, filesize=11.7 K 2024-11-13T00:14:06,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/91ea12042c324ebca6a0dd2a963d81aa as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/91ea12042c324ebca6a0dd2a963d81aa 2024-11-13T00:14:06,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:06,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:06,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/91ea12042c324ebca6a0dd2a963d81aa, entries=150, sequenceid=63, filesize=11.7 K 2024-11-13T00:14:06,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1ee97d494479a45762cab6f40ed6001d in 260ms, sequenceid=63, compaction requested=true 2024-11-13T00:14:06,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:06,408 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:06,409 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:06,410 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:06,410 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:06,410 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,410 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:06,410 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=82.3 K 2024-11-13T00:14:06,410 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:06,410 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,410 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a10fd20937f84303b0565604934796fe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5a2eca25d9014852b3055b3211633014, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bc762089319648a68f38a657238d6f51] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=32.9 K 2024-11-13T00:14:06,410 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,411 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b] 2024-11-13T00:14:06,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a10fd20937f84303b0565604934796fe, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731456844668 2024-11-13T00:14:06,411 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1011b8f222074026b24a53293cd5659e, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731456844668 2024-11-13T00:14:06,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2eca25d9014852b3055b3211633014, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731456844741 2024-11-13T00:14:06,412 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f368467f9f47a8a6731d6bfc1b69b6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731456844741 2024-11-13T00:14:06,412 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bc762089319648a68f38a657238d6f51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731456845459 2024-11-13T00:14:06,412 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52fecda1679b400ba5bb907ed859241b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731456845459 2024-11-13T00:14:06,421 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:06,422 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:06,423 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/c6612dbfcdc944c281f24dc66d900aa1 is 50, key is test_row_0/B:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,428 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111327b81394b49e46018bdf3112c7d086be_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:06,437 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111327b81394b49e46018bdf3112c7d086be_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:06,437 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111327b81394b49e46018bdf3112c7d086be_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:06,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=140.89 KB heapSize=403.63 KB 2024-11-13T00:14:06,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:06,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:06,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:06,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136a97fbd5ffdb43a393eb1fff57938c83_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456846322/Put/seqid=0 2024-11-13T00:14:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456906487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456906487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741973_1149 (size=4469) 2024-11-13T00:14:06,503 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#130 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:06,504 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/bccc526ba7814dac9be4777975142b82 is 175, key is test_row_0/A:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741972_1148 (size=12104) 2024-11-13T00:14:06,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741974_1150 (size=14594) 2024-11-13T00:14:06,520 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:06,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136a97fbd5ffdb43a393eb1fff57938c83_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136a97fbd5ffdb43a393eb1fff57938c83_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:06,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/04c8639fad3b4910910751c6d2d811e5, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:06,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/04c8639fad3b4910910751c6d2d811e5 is 175, key is test_row_0/A:col10/1731456846322/Put/seqid=0 2024-11-13T00:14:06,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741975_1151 (size=31058) 2024-11-13T00:14:06,545 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/bccc526ba7814dac9be4777975142b82 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/bccc526ba7814dac9be4777975142b82 2024-11-13T00:14:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741976_1152 (size=39549) 2024-11-13T00:14:06,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/04c8639fad3b4910910751c6d2d811e5 2024-11-13T00:14:06,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,553 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into bccc526ba7814dac9be4777975142b82(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:06,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,553 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:06,553 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456846408; duration=0sec 2024-11-13T00:14:06,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,553 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:06,553 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:06,553 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,561 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:06,561 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:06,561 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,561 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d58213203244a7dba40af964674ae97, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cc6007d2909046ecba721dff65a0bb67, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/91ea12042c324ebca6a0dd2a963d81aa] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=32.9 K 2024-11-13T00:14:06,562 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d58213203244a7dba40af964674ae97, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731456844668 2024-11-13T00:14:06,562 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc6007d2909046ecba721dff65a0bb67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731456844741 2024-11-13T00:14:06,562 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ea12042c324ebca6a0dd2a963d81aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731456845459 2024-11-13T00:14:06,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a1cfbcd6307d4520bbed902583e7626a is 50, key is test_row_0/B:col10/1731456846322/Put/seqid=0 2024-11-13T00:14:06,586 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:06,586 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/1853a7ada461414380f77e7c95bddbf8 is 50, key is test_row_0/C:col10/1731456845459/Put/seqid=0 2024-11-13T00:14:06,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741977_1153 (size=12001) 2024-11-13T00:14:06,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741978_1154 (size=12104) 2024-11-13T00:14:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456906606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456906606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456906606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456906616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456906636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456906816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456906816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:06,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:06,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:06,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:06,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:06,922 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/c6612dbfcdc944c281f24dc66d900aa1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/c6612dbfcdc944c281f24dc66d900aa1 2024-11-13T00:14:06,936 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into c6612dbfcdc944c281f24dc66d900aa1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:06,936 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:06,936 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456846408; duration=0sec 2024-11-13T00:14:06,936 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:06,936 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:06,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a1cfbcd6307d4520bbed902583e7626a 2024-11-13T00:14:07,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,015 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/1853a7ada461414380f77e7c95bddbf8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1853a7ada461414380f77e7c95bddbf8 2024-11-13T00:14:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/df9889c4e934474f83806485ffa3f67f is 50, key is test_row_0/C:col10/1731456846322/Put/seqid=0 2024-11-13T00:14:07,025 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 1853a7ada461414380f77e7c95bddbf8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:07,026 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:07,026 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456846409; duration=0sec 2024-11-13T00:14:07,026 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:07,026 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:07,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741979_1155 (size=12001) 2024-11-13T00:14:07,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/df9889c4e934474f83806485ffa3f67f 2024-11-13T00:14:07,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/04c8639fad3b4910910751c6d2d811e5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5 2024-11-13T00:14:07,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5, entries=200, sequenceid=95, filesize=38.6 K 2024-11-13T00:14:07,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a1cfbcd6307d4520bbed902583e7626a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1cfbcd6307d4520bbed902583e7626a 2024-11-13T00:14:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1cfbcd6307d4520bbed902583e7626a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T00:14:07,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/df9889c4e934474f83806485ffa3f67f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df9889c4e934474f83806485ffa3f67f 2024-11-13T00:14:07,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df9889c4e934474f83806485ffa3f67f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T00:14:07,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~403.58 KB/413264, currentSize=46.96 KB/48090 for 1ee97d494479a45762cab6f40ed6001d in 648ms, sequenceid=95, compaction requested=false 2024-11-13T00:14:07,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:14:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:07,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:07,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:07,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:07,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:07,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e920eb0df9cb4138b8bf84adace62274_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741980_1156 (size=12154) 2024-11-13T00:14:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456847208, max timestamp=1731456847228Num uniques 859; 2024-11-13T00:14:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456847208, max timestamp=1731456847228Num uniques 859; 2024-11-13T00:14:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456847208, max timestamp=1731456847228Num uniques 859; 2024-11-13T00:14:07,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:07,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456907291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456907297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456907406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456907417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:07,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e920eb0df9cb4138b8bf84adace62274_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e920eb0df9cb4138b8bf84adace62274_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:07,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a299b8e0c608426fb962ac3410d12450, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:07,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a299b8e0c608426fb962ac3410d12450 is 175, key is test_row_0/A:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:07,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741981_1157 (size=30951) 2024-11-13T00:14:07,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a299b8e0c608426fb962ac3410d12450 2024-11-13T00:14:07,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456907616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/f813bebb3efc49619964f9a4ebcec822 is 50, key is test_row_0/B:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:07,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456907626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456907627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,631 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:14:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741982_1158 (size=9657) 2024-11-13T00:14:07,632 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,632 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456907636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456907646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,653 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456907926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:07,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:07,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456907946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/f813bebb3efc49619964f9a4ebcec822 2024-11-13T00:14:08,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/690e2b8805f14e6db303a64ecd90f1ad is 50, key is test_row_0/C:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:08,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741983_1159 (size=9657) 2024-11-13T00:14:08,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/690e2b8805f14e6db303a64ecd90f1ad 2024-11-13T00:14:08,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a299b8e0c608426fb962ac3410d12450 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450 2024-11-13T00:14:08,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450, entries=150, sequenceid=109, filesize=30.2 K 2024-11-13T00:14:08,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/f813bebb3efc49619964f9a4ebcec822 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f813bebb3efc49619964f9a4ebcec822 2024-11-13T00:14:08,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:08,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f813bebb3efc49619964f9a4ebcec822, entries=100, sequenceid=109, filesize=9.4 K 2024-11-13T00:14:08,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/690e2b8805f14e6db303a64ecd90f1ad as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/690e2b8805f14e6db303a64ecd90f1ad 2024-11-13T00:14:08,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/690e2b8805f14e6db303a64ecd90f1ad, entries=100, sequenceid=109, filesize=9.4 K 2024-11-13T00:14:08,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1ee97d494479a45762cab6f40ed6001d in 978ms, sequenceid=109, compaction requested=true 2024-11-13T00:14:08,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:08,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:08,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:08,113 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:08,113 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:08,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:08,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:08,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:08,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:08,114 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101558 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:08,114 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:08,114 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:08,115 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,115 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:08,115 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,115 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/bccc526ba7814dac9be4777975142b82, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=99.2 K 2024-11-13T00:14:08,115 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/c6612dbfcdc944c281f24dc66d900aa1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1cfbcd6307d4520bbed902583e7626a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f813bebb3efc49619964f9a4ebcec822] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=33.0 K 2024-11-13T00:14:08,115 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,115 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/bccc526ba7814dac9be4777975142b82, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450] 2024-11-13T00:14:08,116 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting c6612dbfcdc944c281f24dc66d900aa1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731456845459 2024-11-13T00:14:08,116 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting bccc526ba7814dac9be4777975142b82, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731456845459 2024-11-13T00:14:08,116 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a1cfbcd6307d4520bbed902583e7626a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731456846321 2024-11-13T00:14:08,116 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04c8639fad3b4910910751c6d2d811e5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731456846293 2024-11-13T00:14:08,117 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a299b8e0c608426fb962ac3410d12450, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731456846484 2024-11-13T00:14:08,117 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f813bebb3efc49619964f9a4ebcec822, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731456846484 2024-11-13T00:14:08,139 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:08,139 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8874a19ea2ad41cba61b10cb0d506b63 is 50, key is test_row_0/B:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:08,142 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:08,144 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132bf20b604ff044beab1b63411e18e492_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:08,147 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132bf20b604ff044beab1b63411e18e492_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:08,147 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132bf20b604ff044beab1b63411e18e492_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741985_1161 (size=4469) 2024-11-13T00:14:08,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741984_1160 (size=12207) 2024-11-13T00:14:08,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T00:14:08,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,255 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T00:14:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:08,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323e49a202cad4f73a05065dd878c2ab1_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456847294/Put/seqid=0 2024-11-13T00:14:08,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741986_1162 (size=12304) 2024-11-13T00:14:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,292 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323e49a202cad4f73a05065dd878c2ab1_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323e49a202cad4f73a05065dd878c2ab1_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/6a76ab7ba0994bd182550a88ed375e42, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/6a76ab7ba0994bd182550a88ed375e42 is 175, key is test_row_0/A:col10/1731456847294/Put/seqid=0 2024-11-13T00:14:08,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741987_1163 (size=31105) 2024-11-13T00:14:08,357 INFO [master/cb28580903fb:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T00:14:08,357 INFO [master/cb28580903fb:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T00:14:08,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:08,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456908485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:08,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456908485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,558 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:08,559 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/60e080cf96004d83bf927d411dbcb9a2 is 175, key is test_row_0/A:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:08,572 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8874a19ea2ad41cba61b10cb0d506b63 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8874a19ea2ad41cba61b10cb0d506b63 2024-11-13T00:14:08,579 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into 8874a19ea2ad41cba61b10cb0d506b63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:08,579 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:08,579 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456848113; duration=0sec 2024-11-13T00:14:08,579 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:08,582 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:08,582 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:08,583 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:08,583 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:08,583 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,583 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1853a7ada461414380f77e7c95bddbf8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df9889c4e934474f83806485ffa3f67f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/690e2b8805f14e6db303a64ecd90f1ad] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=33.0 K 2024-11-13T00:14:08,584 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1853a7ada461414380f77e7c95bddbf8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731456845459 2024-11-13T00:14:08,584 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting df9889c4e934474f83806485ffa3f67f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731456846321 2024-11-13T00:14:08,585 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 690e2b8805f14e6db303a64ecd90f1ad, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731456846484 2024-11-13T00:14:08,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741988_1164 (size=31268) 2024-11-13T00:14:08,605 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/60e080cf96004d83bf927d411dbcb9a2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/60e080cf96004d83bf927d411dbcb9a2 2024-11-13T00:14:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456908597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:08,607 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:08,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456908597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,607 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/53c4e399f8a4409da79a3397452931fe is 50, key is test_row_0/C:col10/1731456846484/Put/seqid=0 2024-11-13T00:14:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,618 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into 60e080cf96004d83bf927d411dbcb9a2(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:08,618 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:08,618 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456848113; duration=0sec 2024-11-13T00:14:08,618 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:08,619 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741989_1165 (size=12207) 2024-11-13T00:14:08,638 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/53c4e399f8a4409da79a3397452931fe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/53c4e399f8a4409da79a3397452931fe 2024-11-13T00:14:08,652 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 53c4e399f8a4409da79a3397452931fe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:08,652 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:08,652 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456848114; duration=0sec 2024-11-13T00:14:08,652 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:08,652 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:08,703 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/6a76ab7ba0994bd182550a88ed375e42 2024-11-13T00:14:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/008e306178284858b5c5915a6ffbca21 is 50, key is test_row_0/B:col10/1731456847294/Put/seqid=0 2024-11-13T00:14:08,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741990_1166 (size=12151) 2024-11-13T00:14:08,725 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/008e306178284858b5c5915a6ffbca21 2024-11-13T00:14:08,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/ee5cd254b728483e87809939abc66ea2 is 50, key is test_row_0/C:col10/1731456847294/Put/seqid=0 2024-11-13T00:14:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741991_1167 (size=12151) 2024-11-13T00:14:08,755 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/ee5cd254b728483e87809939abc66ea2 2024-11-13T00:14:08,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/6a76ab7ba0994bd182550a88ed375e42 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42 2024-11-13T00:14:08,773 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42, entries=150, sequenceid=144, filesize=30.4 K 2024-11-13T00:14:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/008e306178284858b5c5915a6ffbca21 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/008e306178284858b5c5915a6ffbca21 2024-11-13T00:14:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,783 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/008e306178284858b5c5915a6ffbca21, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T00:14:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/ee5cd254b728483e87809939abc66ea2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ee5cd254b728483e87809939abc66ea2 2024-11-13T00:14:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,791 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ee5cd254b728483e87809939abc66ea2, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T00:14:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,793 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1ee97d494479a45762cab6f40ed6001d in 538ms, sequenceid=144, compaction requested=false 2024-11-13T00:14:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T00:14:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T00:14:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T00:14:08,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1070 sec 2024-11-13T00:14:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 4.1150 sec 2024-11-13T00:14:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:08,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:14:08,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:08,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:08,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:08,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:08,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c0892c5304ae4f8fb1cca9ebf57fe18f_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741992_1168 (size=12304) 2024-11-13T00:14:08,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T00:14:08,868 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:08,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:08,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c0892c5304ae4f8fb1cca9ebf57fe18f_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c0892c5304ae4f8fb1cca9ebf57fe18f_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:08,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/d8fd3456fc66459a9c7a78dd410e854e, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:08,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/d8fd3456fc66459a9c7a78dd410e854e is 175, key is test_row_0/A:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T00:14:08,875 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T00:14:08,876 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:08,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:08,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741993_1169 (size=31105) 2024-11-13T00:14:08,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/d8fd3456fc66459a9c7a78dd410e854e 2024-11-13T00:14:08,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:08,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/b32e65b7c6a742788f322b992bb503b4 is 50, key is test_row_0/B:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456908907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:08,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741994_1170 (size=12151) 2024-11-13T00:14:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456908921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:08,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/b32e65b7c6a742788f322b992bb503b4 2024-11-13T00:14:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:08,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/a87c3f628bc843e8b48f3d7b834c5f5b is 50, key is test_row_0/C:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741995_1171 (size=12151) 2024-11-13T00:14:08,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/a87c3f628bc843e8b48f3d7b834c5f5b 2024-11-13T00:14:08,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/d8fd3456fc66459a9c7a78dd410e854e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e 2024-11-13T00:14:08,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e, entries=150, sequenceid=159, filesize=30.4 K 2024-11-13T00:14:08,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/b32e65b7c6a742788f322b992bb503b4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/b32e65b7c6a742788f322b992bb503b4 2024-11-13T00:14:08,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/b32e65b7c6a742788f322b992bb503b4, entries=150, sequenceid=159, filesize=11.9 K 2024-11-13T00:14:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T00:14:08,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/a87c3f628bc843e8b48f3d7b834c5f5b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a87c3f628bc843e8b48f3d7b834c5f5b 2024-11-13T00:14:08,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a87c3f628bc843e8b48f3d7b834c5f5b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-13T00:14:08,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1ee97d494479a45762cab6f40ed6001d in 169ms, sequenceid=159, compaction requested=true 2024-11-13T00:14:08,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:08,992 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:08,993 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93478 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:08,993 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:08,993 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,994 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/60e080cf96004d83bf927d411dbcb9a2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=91.3 K 2024-11-13T00:14:08,994 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,994 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/60e080cf96004d83bf927d411dbcb9a2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e] 2024-11-13T00:14:08,994 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60e080cf96004d83bf927d411dbcb9a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731456846441 2024-11-13T00:14:08,995 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a76ab7ba0994bd182550a88ed375e42, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731456847261 2024-11-13T00:14:08,995 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fd3456fc66459a9c7a78dd410e854e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731456848468 2024-11-13T00:14:08,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:08,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:08,996 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:08,998 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:08,998 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:08,998 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:08,998 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8874a19ea2ad41cba61b10cb0d506b63, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/008e306178284858b5c5915a6ffbca21, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/b32e65b7c6a742788f322b992bb503b4] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=35.7 K 2024-11-13T00:14:08,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:08,999 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8874a19ea2ad41cba61b10cb0d506b63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731456846441 2024-11-13T00:14:08,999 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 008e306178284858b5c5915a6ffbca21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731456847261 2024-11-13T00:14:08,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:09,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:09,000 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b32e65b7c6a742788f322b992bb503b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731456848468 2024-11-13T00:14:09,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:09,007 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:09,010 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111359732578a9d84c3a976c1b78947d2853_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:09,013 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111359732578a9d84c3a976c1b78947d2853_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:09,013 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359732578a9d84c3a976c1b78947d2853_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:09,020 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:09,020 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/9fbf385de1344666b0a423cd7a641441 is 50, key is test_row_0/B:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:09,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456848888, max timestamp=1731456849019Num uniques 859; 2024-11-13T00:14:09,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456848888, max timestamp=1731456849019Num uniques 859; 2024-11-13T00:14:09,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:09,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:09,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456848888, max timestamp=1731456849019Num uniques 859; 2024-11-13T00:14:09,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:09,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T00:14:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:09,034 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-13T00:14:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:09,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:09,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:09,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5e172ab3bf34d33ae72ce9f131d684a_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456848905/Put/seqid=0 2024-11-13T00:14:09,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741996_1172 (size=4469) 2024-11-13T00:14:09,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741997_1173 (size=12459) 2024-11-13T00:14:09,076 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/9fbf385de1344666b0a423cd7a641441 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9fbf385de1344666b0a423cd7a641441 2024-11-13T00:14:09,086 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into 9fbf385de1344666b0a423cd7a641441(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:09,086 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:09,086 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456848996; duration=0sec 2024-11-13T00:14:09,086 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:09,086 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:09,086 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:09,087 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:09,088 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:09,088 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:09,089 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/53c4e399f8a4409da79a3397452931fe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ee5cd254b728483e87809939abc66ea2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a87c3f628bc843e8b48f3d7b834c5f5b] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=35.7 K 2024-11-13T00:14:09,090 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c4e399f8a4409da79a3397452931fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731456846441 2024-11-13T00:14:09,090 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5cd254b728483e87809939abc66ea2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731456847261 2024-11-13T00:14:09,091 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a87c3f628bc843e8b48f3d7b834c5f5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731456848468 2024-11-13T00:14:09,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741998_1174 (size=12304) 2024-11-13T00:14:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:09,111 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#150 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:09,112 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/df75422792b54540ac93f14edd61a58b is 50, key is test_row_0/C:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:09,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456849080, max timestamp=1731456849110Num uniques 859; 2024-11-13T00:14:09,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:09,114 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5e172ab3bf34d33ae72ce9f131d684a_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5e172ab3bf34d33ae72ce9f131d684a_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4fb67bdbf5354ddd825a957d738ddbad, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:09,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4fb67bdbf5354ddd825a957d738ddbad is 175, key is test_row_0/A:col10/1731456848905/Put/seqid=0 2024-11-13T00:14:09,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456849080, max timestamp=1731456849110Num uniques 859; 2024-11-13T00:14:09,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:09,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456849080, max timestamp=1731456849110Num uniques 859; 2024-11-13T00:14:09,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:09,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741999_1175 (size=31105) 2024-11-13T00:14:09,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742000_1176 (size=12459) 2024-11-13T00:14:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T00:14:09,198 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/df75422792b54540ac93f14edd61a58b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df75422792b54540ac93f14edd61a58b 2024-11-13T00:14:09,215 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into df75422792b54540ac93f14edd61a58b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:09,215 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:09,215 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456849000; duration=0sec 2024-11-13T00:14:09,215 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:09,215 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:09,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731456849146, max timestamp=1731456849223Num uniques 1068; 2024-11-13T00:14:09,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731456849146, max timestamp=1731456849223Num uniques 1068; 2024-11-13T00:14:09,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731456849146, max timestamp=1731456849223Num uniques 1068; 2024-11-13T00:14:09,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T00:14:09,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T00:14:09,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T00:14:09,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456909301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456909308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456909416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456909416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,465 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:09,466 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/9237d423e6514a05ab98652aa4fdac62 is 175, key is test_row_0/A:col10/1731456848468/Put/seqid=0 2024-11-13T00:14:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742001_1177 (size=31413) 2024-11-13T00:14:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T00:14:09,559 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4fb67bdbf5354ddd825a957d738ddbad 2024-11-13T00:14:09,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/37b83f56a30244959e2403af2e0ccdac is 50, key is test_row_0/B:col10/1731456848905/Put/seqid=0 2024-11-13T00:14:09,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742002_1178 (size=12151) 2024-11-13T00:14:09,604 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/37b83f56a30244959e2403af2e0ccdac 2024-11-13T00:14:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/9029d02c8d5542e4b80bf34196100b6a is 50, key is test_row_0/C:col10/1731456848905/Put/seqid=0 2024-11-13T00:14:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456909636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456909637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456909646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456909646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,649 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,648 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742003_1179 (size=12151) 2024-11-13T00:14:09,662 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/9029d02c8d5542e4b80bf34196100b6a 2024-11-13T00:14:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4fb67bdbf5354ddd825a957d738ddbad as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad 2024-11-13T00:14:09,676 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad, entries=150, sequenceid=182, filesize=30.4 K 2024-11-13T00:14:09,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456909676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:09,678 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:09,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/37b83f56a30244959e2403af2e0ccdac as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/37b83f56a30244959e2403af2e0ccdac 2024-11-13T00:14:09,686 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/37b83f56a30244959e2403af2e0ccdac, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T00:14:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/9029d02c8d5542e4b80bf34196100b6a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/9029d02c8d5542e4b80bf34196100b6a 2024-11-13T00:14:09,695 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/9029d02c8d5542e4b80bf34196100b6a, entries=150, sequenceid=182, filesize=11.9 K 2024-11-13T00:14:09,697 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 1ee97d494479a45762cab6f40ed6001d in 663ms, sequenceid=182, compaction requested=false 2024-11-13T00:14:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T00:14:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T00:14:09,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T00:14:09,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 824 msec 2024-11-13T00:14:09,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 835 msec 2024-11-13T00:14:09,900 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/9237d423e6514a05ab98652aa4fdac62 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/9237d423e6514a05ab98652aa4fdac62 2024-11-13T00:14:09,907 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into 9237d423e6514a05ab98652aa4fdac62(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:09,908 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:09,908 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456848992; duration=0sec 2024-11-13T00:14:09,908 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:09,908 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:09,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731456849282, max timestamp=1731456849947Num uniques 1127; 2024-11-13T00:14:09,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-13T00:14:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:09,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=111.82 KB heapSize=311.67 KB 2024-11-13T00:14:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:09,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113704a87a4300f402885d64204a5327c91_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:09,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742004_1180 (size=14794) 2024-11-13T00:14:09,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:09,989 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113704a87a4300f402885d64204a5327c91_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113704a87a4300f402885d64204a5327c91_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:09,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/ae441610d70e4c7f9927da72c4ebdce3, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:09,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/ae441610d70e4c7f9927da72c4ebdce3 is 175, key is test_row_0/A:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:10,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456909997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742005_1181 (size=39749) 2024-11-13T00:14:10,005 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/ae441610d70e4c7f9927da72c4ebdce3 2024-11-13T00:14:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456910002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8c0b2e49ca6249d2b8e59910fdc794a0 is 50, key is test_row_0/B:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T00:14:10,016 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:10,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-13T00:14:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T00:14:10,019 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:10,019 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:10,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:10,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742006_1182 (size=12151) 2024-11-13T00:14:10,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456910106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456910116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T00:14:10,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T00:14:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:10,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456910316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T00:14:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:10,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456910326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T00:14:10,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8c0b2e49ca6249d2b8e59910fdc794a0 2024-11-13T00:14:10,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/bff4ce06eff34c4a9000a870cd5be11f is 50, key is test_row_0/C:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:10,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742007_1183 (size=12151) 2024-11-13T00:14:10,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/bff4ce06eff34c4a9000a870cd5be11f 2024-11-13T00:14:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/ae441610d70e4c7f9927da72c4ebdce3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3 2024-11-13T00:14:10,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3, entries=200, sequenceid=240, filesize=38.8 K 2024-11-13T00:14:10,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8c0b2e49ca6249d2b8e59910fdc794a0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8c0b2e49ca6249d2b8e59910fdc794a0 2024-11-13T00:14:10,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8c0b2e49ca6249d2b8e59910fdc794a0, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T00:14:10,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/bff4ce06eff34c4a9000a870cd5be11f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bff4ce06eff34c4a9000a870cd5be11f 2024-11-13T00:14:10,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bff4ce06eff34c4a9000a870cd5be11f, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T00:14:10,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~311.63 KB/319104, currentSize=73.80 KB/75570 for 1ee97d494479a45762cab6f40ed6001d in 523ms, sequenceid=240, compaction requested=true 2024-11-13T00:14:10,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:10,475 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:10,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:10,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:10,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:10,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:10,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:10,477 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:10,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:10,479 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:10,479 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:10,479 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,479 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/9237d423e6514a05ab98652aa4fdac62, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=99.9 K 2024-11-13T00:14:10,479 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,479 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/9237d423e6514a05ab98652aa4fdac62, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3] 2024-11-13T00:14:10,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T00:14:10,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,480 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:14:10,480 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9237d423e6514a05ab98652aa4fdac62, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731456848468 2024-11-13T00:14:10,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:10,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:10,481 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:10,481 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:10,481 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,482 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9fbf385de1344666b0a423cd7a641441, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/37b83f56a30244959e2403af2e0ccdac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8c0b2e49ca6249d2b8e59910fdc794a0] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=35.9 K 2024-11-13T00:14:10,482 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fb67bdbf5354ddd825a957d738ddbad, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731456848905 2024-11-13T00:14:10,482 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbf385de1344666b0a423cd7a641441, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731456848468 2024-11-13T00:14:10,482 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae441610d70e4c7f9927da72c4ebdce3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731456849282 2024-11-13T00:14:10,484 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b83f56a30244959e2403af2e0ccdac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731456848905 2024-11-13T00:14:10,485 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0b2e49ca6249d2b8e59910fdc794a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731456849298 2024-11-13T00:14:10,498 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:10,506 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:10,507 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/bbeb3d0a3fdb45ed9461b51d4de5e865 is 50, key is test_row_0/B:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:10,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131ad679e885a3431191013bdf50176a1d_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456850001/Put/seqid=0 2024-11-13T00:14:10,512 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135a809ed665864b04ad10aa8070f242fc_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:10,514 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135a809ed665864b04ad10aa8070f242fc_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:10,514 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135a809ed665864b04ad10aa8070f242fc_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:10,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742009_1185 (size=12304) 2024-11-13T00:14:10,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742010_1186 (size=4469) 2024-11-13T00:14:10,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742008_1184 (size=12561) 2024-11-13T00:14:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:10,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T00:14:10,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456910704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456910707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456910816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456910826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:10,934 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131ad679e885a3431191013bdf50176a1d_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131ad679e885a3431191013bdf50176a1d_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:10,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4ea8f519c1b94104ba9462c8abb40c65, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:10,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4ea8f519c1b94104ba9462c8abb40c65 is 175, key is test_row_0/A:col10/1731456850001/Put/seqid=0 2024-11-13T00:14:10,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742011_1187 (size=31105) 2024-11-13T00:14:10,947 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:10,947 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/6e210f0b5cb54e318faffd39de07096d is 175, key is test_row_0/A:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:10,963 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/bbeb3d0a3fdb45ed9461b51d4de5e865 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bbeb3d0a3fdb45ed9461b51d4de5e865 2024-11-13T00:14:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742012_1188 (size=31515) 2024-11-13T00:14:10,976 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into bbeb3d0a3fdb45ed9461b51d4de5e865(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:10,976 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:10,976 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456850476; duration=0sec 2024-11-13T00:14:10,977 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:10,977 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:10,977 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:10,985 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:10,985 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:10,985 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:10,986 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df75422792b54540ac93f14edd61a58b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/9029d02c8d5542e4b80bf34196100b6a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bff4ce06eff34c4a9000a870cd5be11f] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=35.9 K 2024-11-13T00:14:10,986 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting df75422792b54540ac93f14edd61a58b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731456848468 2024-11-13T00:14:10,987 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9029d02c8d5542e4b80bf34196100b6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731456848905 2024-11-13T00:14:10,988 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bff4ce06eff34c4a9000a870cd5be11f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731456849298 2024-11-13T00:14:10,991 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/6e210f0b5cb54e318faffd39de07096d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6e210f0b5cb54e318faffd39de07096d 2024-11-13T00:14:11,000 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into 6e210f0b5cb54e318faffd39de07096d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:11,000 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:11,000 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456850475; duration=0sec 2024-11-13T00:14:11,001 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:11,001 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:11,011 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:11,017 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/44ee50b01536456c9daea41116743ded is 50, key is test_row_0/C:col10/1731456849947/Put/seqid=0 2024-11-13T00:14:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456911027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:11,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456911036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:11,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742013_1189 (size=12561) 2024-11-13T00:14:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T00:14:11,347 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4ea8f519c1b94104ba9462c8abb40c65 2024-11-13T00:14:11,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456911346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:11,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456911346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:11,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a1f8740fa55c4f7aad2930dac51962d8 is 50, key is test_row_0/B:col10/1731456850001/Put/seqid=0 2024-11-13T00:14:11,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742014_1190 (size=12151) 2024-11-13T00:14:11,385 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a1f8740fa55c4f7aad2930dac51962d8 2024-11-13T00:14:11,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/a5b86e953390443889e68af8728a52a3 is 50, key is test_row_0/C:col10/1731456850001/Put/seqid=0 2024-11-13T00:14:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742015_1191 (size=12151) 2024-11-13T00:14:11,412 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/a5b86e953390443889e68af8728a52a3 2024-11-13T00:14:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/4ea8f519c1b94104ba9462c8abb40c65 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65 2024-11-13T00:14:11,424 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65, entries=150, sequenceid=254, filesize=30.4 K 2024-11-13T00:14:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a1f8740fa55c4f7aad2930dac51962d8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1f8740fa55c4f7aad2930dac51962d8 2024-11-13T00:14:11,435 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1f8740fa55c4f7aad2930dac51962d8, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T00:14:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/a5b86e953390443889e68af8728a52a3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a5b86e953390443889e68af8728a52a3 2024-11-13T00:14:11,440 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a5b86e953390443889e68af8728a52a3, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T00:14:11,442 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1ee97d494479a45762cab6f40ed6001d in 961ms, sequenceid=254, compaction requested=false 2024-11-13T00:14:11,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:11,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T00:14:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T00:14:11,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T00:14:11,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-13T00:14:11,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4350 sec 2024-11-13T00:14:11,476 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/44ee50b01536456c9daea41116743ded as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/44ee50b01536456c9daea41116743ded 2024-11-13T00:14:11,485 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 44ee50b01536456c9daea41116743ded(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:11,485 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:11,485 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456850477; duration=0sec 2024-11-13T00:14:11,485 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:11,485 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:11,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:14:11,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:11,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:11,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:11,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:11,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:11,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:11,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136977205d40cb4a6ba919fd65df561d0c_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:11,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742016_1192 (size=12454) 2024-11-13T00:14:11,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:11,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456911894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:11,902 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136977205d40cb4a6ba919fd65df561d0c_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136977205d40cb4a6ba919fd65df561d0c_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:11,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/48e888d26543429f8e7dcb09d39d0929, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:11,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/48e888d26543429f8e7dcb09d39d0929 is 175, key is test_row_0/A:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456911896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:11,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742017_1193 (size=31255) 2024-11-13T00:14:11,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/48e888d26543429f8e7dcb09d39d0929 2024-11-13T00:14:11,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/d82fead44d7b4931ab52e956878623b0 is 50, key is test_row_0/B:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:11,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742018_1194 (size=12301) 2024-11-13T00:14:11,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/d82fead44d7b4931ab52e956878623b0 2024-11-13T00:14:11,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/aa87d54bcabc4456b54e2598eb879385 is 50, key is test_row_0/C:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:11,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742019_1195 (size=12301) 2024-11-13T00:14:11,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/aa87d54bcabc4456b54e2598eb879385 2024-11-13T00:14:11,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/48e888d26543429f8e7dcb09d39d0929 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929 2024-11-13T00:14:11,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929, entries=150, sequenceid=277, filesize=30.5 K 2024-11-13T00:14:11,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/d82fead44d7b4931ab52e956878623b0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d82fead44d7b4931ab52e956878623b0 2024-11-13T00:14:12,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d82fead44d7b4931ab52e956878623b0, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T00:14:12,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/aa87d54bcabc4456b54e2598eb879385 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/aa87d54bcabc4456b54e2598eb879385 2024-11-13T00:14:12,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456912006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/aa87d54bcabc4456b54e2598eb879385, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T00:14:12,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 1ee97d494479a45762cab6f40ed6001d in 155ms, sequenceid=277, compaction requested=true 2024-11-13T00:14:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:12,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:12,014 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:12,014 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:12,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:12,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:12,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:12,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:12,016 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93875 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:12,016 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:12,016 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,016 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6e210f0b5cb54e318faffd39de07096d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=91.7 K 2024-11-13T00:14:12,016 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,016 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6e210f0b5cb54e318faffd39de07096d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929] 2024-11-13T00:14:12,016 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:12,017 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:12,017 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,017 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bbeb3d0a3fdb45ed9461b51d4de5e865, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1f8740fa55c4f7aad2930dac51962d8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d82fead44d7b4931ab52e956878623b0] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.1 K 2024-11-13T00:14:12,017 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e210f0b5cb54e318faffd39de07096d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731456849298 2024-11-13T00:14:12,019 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bbeb3d0a3fdb45ed9461b51d4de5e865, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731456849298 2024-11-13T00:14:12,019 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea8f519c1b94104ba9462c8abb40c65, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731456849969 2024-11-13T00:14:12,019 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f8740fa55c4f7aad2930dac51962d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731456849969 2024-11-13T00:14:12,019 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e888d26543429f8e7dcb09d39d0929, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731456850701 2024-11-13T00:14:12,019 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d82fead44d7b4931ab52e956878623b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731456850701 2024-11-13T00:14:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:12,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:14:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:12,027 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:12,030 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:12,031 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8ad810db0d8c46bb954c2afed063f7a4 is 50, key is test_row_0/B:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:12,043 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113aa24e5e9cedd4d12bfb8ca4eea5dc3be_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:12,045 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113aa24e5e9cedd4d12bfb8ca4eea5dc3be_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:12,045 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa24e5e9cedd4d12bfb8ca4eea5dc3be_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:12,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134fb76c809a2f4d22b3fee835b63d74b0_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456851892/Put/seqid=0 2024-11-13T00:14:12,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742020_1196 (size=12813) 2024-11-13T00:14:12,074 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/8ad810db0d8c46bb954c2afed063f7a4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8ad810db0d8c46bb954c2afed063f7a4 2024-11-13T00:14:12,081 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into 8ad810db0d8c46bb954c2afed063f7a4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:12,081 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:12,081 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456852014; duration=0sec 2024-11-13T00:14:12,081 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:12,081 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:12,081 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:12,082 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:12,082 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:12,083 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,083 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/44ee50b01536456c9daea41116743ded, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a5b86e953390443889e68af8728a52a3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/aa87d54bcabc4456b54e2598eb879385] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.1 K 2024-11-13T00:14:12,085 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ee50b01536456c9daea41116743ded, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731456849298 2024-11-13T00:14:12,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742021_1197 (size=4469) 2024-11-13T00:14:12,087 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b86e953390443889e68af8728a52a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731456849969 2024-11-13T00:14:12,088 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#165 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:12,088 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting aa87d54bcabc4456b54e2598eb879385, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731456850701 2024-11-13T00:14:12,088 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/dc781b8275ae466dbcb6221ab59d16bf is 175, key is test_row_0/A:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:12,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742022_1198 (size=14994) 2024-11-13T00:14:12,112 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:12,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134fb76c809a2f4d22b3fee835b63d74b0_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134fb76c809a2f4d22b3fee835b63d74b0_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:12,120 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8fc603e611c647e8908e4d92beb21783, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:12,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8fc603e611c647e8908e4d92beb21783 is 175, key is test_row_0/A:col10/1731456851892/Put/seqid=0 2024-11-13T00:14:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456912117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,127 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:12,128 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/d7ea609e0e42479c84cba18b15df3b51 is 50, key is test_row_0/C:col10/1731456851858/Put/seqid=0 2024-11-13T00:14:12,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742023_1199 (size=31767) 2024-11-13T00:14:12,156 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/dc781b8275ae466dbcb6221ab59d16bf as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/dc781b8275ae466dbcb6221ab59d16bf 2024-11-13T00:14:12,163 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into dc781b8275ae466dbcb6221ab59d16bf(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:12,163 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:12,163 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456852014; duration=0sec 2024-11-13T00:14:12,164 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:12,164 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:12,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742024_1200 (size=39949) 2024-11-13T00:14:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T00:14:12,166 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:12,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-13T00:14:12,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8fc603e611c647e8908e4d92beb21783 2024-11-13T00:14:12,169 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T00:14:12,169 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:12,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742025_1201 (size=12813) 2024-11-13T00:14:12,187 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/d7ea609e0e42479c84cba18b15df3b51 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/d7ea609e0e42479c84cba18b15df3b51 2024-11-13T00:14:12,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3be6ee4e830c4e64af6d5e24a96e7ce1 is 50, key is test_row_0/B:col10/1731456851892/Put/seqid=0 2024-11-13T00:14:12,194 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into d7ea609e0e42479c84cba18b15df3b51(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:12,194 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:12,194 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456852015; duration=0sec 2024-11-13T00:14:12,195 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:12,195 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:12,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456912216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742026_1202 (size=12301) 2024-11-13T00:14:12,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456912226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T00:14:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456912436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T00:14:12,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456912525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3be6ee4e830c4e64af6d5e24a96e7ce1 2024-11-13T00:14:12,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/48bd1222ab6747d8b82cb6b858a65ddd is 50, key is test_row_0/C:col10/1731456851892/Put/seqid=0 2024-11-13T00:14:12,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742027_1203 (size=12301) 2024-11-13T00:14:12,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/48bd1222ab6747d8b82cb6b858a65ddd 2024-11-13T00:14:12,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8fc603e611c647e8908e4d92beb21783 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783 2024-11-13T00:14:12,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783, entries=200, sequenceid=291, filesize=39.0 K 2024-11-13T00:14:12,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3be6ee4e830c4e64af6d5e24a96e7ce1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3be6ee4e830c4e64af6d5e24a96e7ce1 2024-11-13T00:14:12,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3be6ee4e830c4e64af6d5e24a96e7ce1, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T00:14:12,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/48bd1222ab6747d8b82cb6b858a65ddd as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/48bd1222ab6747d8b82cb6b858a65ddd 2024-11-13T00:14:12,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/48bd1222ab6747d8b82cb6b858a65ddd, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T00:14:12,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1ee97d494479a45762cab6f40ed6001d in 718ms, sequenceid=291, compaction requested=false 2024-11-13T00:14:12,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:12,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:14:12,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:12,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139cc70d38de124f2a9b233111ee1b27bd_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:12,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T00:14:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742028_1204 (size=14994) 2024-11-13T00:14:12,812 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:12,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139cc70d38de124f2a9b233111ee1b27bd_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139cc70d38de124f2a9b233111ee1b27bd_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:12,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/5a178a87e6594eedbcc24928854adb39, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:12,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/5a178a87e6594eedbcc24928854adb39 is 175, key is test_row_0/A:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:12,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456912813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742029_1205 (size=39949) 2024-11-13T00:14:12,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:12,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456912936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:12,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456913046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456913156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:13,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/5a178a87e6594eedbcc24928854adb39 2024-11-13T00:14:13,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/49bb4490ef2347558a95b334828fb54a is 50, key is test_row_0/B:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:13,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742030_1206 (size=12301) 2024-11-13T00:14:13,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/49bb4490ef2347558a95b334828fb54a 2024-11-13T00:14:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T00:14:13,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/33887b4faf384e348d1218c8ad3a3bab is 50, key is test_row_0/C:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:13,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742031_1207 (size=12301) 2024-11-13T00:14:13,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/33887b4faf384e348d1218c8ad3a3bab 2024-11-13T00:14:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/5a178a87e6594eedbcc24928854adb39 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39 2024-11-13T00:14:13,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39, entries=200, sequenceid=313, filesize=39.0 K 2024-11-13T00:14:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/49bb4490ef2347558a95b334828fb54a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/49bb4490ef2347558a95b334828fb54a 2024-11-13T00:14:13,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/49bb4490ef2347558a95b334828fb54a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T00:14:13,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/33887b4faf384e348d1218c8ad3a3bab as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/33887b4faf384e348d1218c8ad3a3bab 2024-11-13T00:14:13,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/33887b4faf384e348d1218c8ad3a3bab, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T00:14:13,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1ee97d494479a45762cab6f40ed6001d in 635ms, sequenceid=313, compaction requested=true 2024-11-13T00:14:13,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:13,389 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:13,389 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:13,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:13,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:13,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:13,390 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:13,390 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:13,390 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,390 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/dc781b8275ae466dbcb6221ab59d16bf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=109.0 K 2024-11-13T00:14:13,390 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/dc781b8275ae466dbcb6221ab59d16bf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39] 2024-11-13T00:14:13,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:13,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:13,391 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,391 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8ad810db0d8c46bb954c2afed063f7a4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3be6ee4e830c4e64af6d5e24a96e7ce1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/49bb4490ef2347558a95b334828fb54a] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.5 K 2024-11-13T00:14:13,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc781b8275ae466dbcb6221ab59d16bf, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731456850701 2024-11-13T00:14:13,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad810db0d8c46bb954c2afed063f7a4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731456850701 2024-11-13T00:14:13,392 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc603e611c647e8908e4d92beb21783, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731456851872 2024-11-13T00:14:13,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3be6ee4e830c4e64af6d5e24a96e7ce1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731456851872 2024-11-13T00:14:13,392 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a178a87e6594eedbcc24928854adb39, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731456852021 2024-11-13T00:14:13,393 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 49bb4490ef2347558a95b334828fb54a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731456852021 2024-11-13T00:14:13,404 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:13,404 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/33dbe666d2cd4a67bcf3bae376d7c39a is 50, key is test_row_0/B:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:13,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T00:14:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,408 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:14:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:13,419 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:13,432 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113dc06b113d4bc40ee9a812f485268dcd0_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:13,434 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113dc06b113d4bc40ee9a812f485268dcd0_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:13,435 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc06b113d4bc40ee9a812f485268dcd0_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:13,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113571fabed26404850813992f0d58f6705_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456852805/Put/seqid=0 2024-11-13T00:14:13,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742032_1208 (size=12915) 2024-11-13T00:14:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:13,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:13,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742034_1210 (size=12454) 2024-11-13T00:14:13,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:13,500 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113571fabed26404850813992f0d58f6705_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113571fabed26404850813992f0d58f6705_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/699eeba2448d45938c709fea57b4ef37, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/699eeba2448d45938c709fea57b4ef37 is 175, key is test_row_0/A:col10/1731456852805/Put/seqid=0 2024-11-13T00:14:13,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742033_1209 (size=4469) 2024-11-13T00:14:13,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742035_1211 (size=31255) 2024-11-13T00:14:13,510 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/699eeba2448d45938c709fea57b4ef37 2024-11-13T00:14:13,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/473c92f09ac44630b975937be628f435 is 50, key is test_row_0/B:col10/1731456852805/Put/seqid=0 2024-11-13T00:14:13,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742036_1212 (size=12301) 2024-11-13T00:14:13,526 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/473c92f09ac44630b975937be628f435 2024-11-13T00:14:13,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/b807eb990f8f454ba7def0c69885586a is 50, key is test_row_0/C:col10/1731456852805/Put/seqid=0 2024-11-13T00:14:13,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742037_1213 (size=12301) 2024-11-13T00:14:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456913566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456913676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456913676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,680 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456913686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,691 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456913715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,724 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,877 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/33dbe666d2cd4a67bcf3bae376d7c39a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/33dbe666d2cd4a67bcf3bae376d7c39a 2024-11-13T00:14:13,886 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into 33dbe666d2cd4a67bcf3bae376d7c39a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:13,886 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:13,886 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456853389; duration=0sec 2024-11-13T00:14:13,886 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:13,886 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:13,886 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:13,887 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:13,887 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:13,887 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,887 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/d7ea609e0e42479c84cba18b15df3b51, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/48bd1222ab6747d8b82cb6b858a65ddd, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/33887b4faf384e348d1218c8ad3a3bab] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.5 K 2024-11-13T00:14:13,888 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ea609e0e42479c84cba18b15df3b51, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731456850701 2024-11-13T00:14:13,888 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 48bd1222ab6747d8b82cb6b858a65ddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731456851872 2024-11-13T00:14:13,889 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 33887b4faf384e348d1218c8ad3a3bab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731456852021 2024-11-13T00:14:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456913886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:13,898 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:13,898 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/bc5ba7f7129b4c61a09d17086792ead0 is 50, key is test_row_0/C:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:13,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742038_1214 (size=12915) 2024-11-13T00:14:13,904 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#175 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:13,905 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/ec27e49b63cc4437a26656100a2f3026 is 175, key is test_row_0/A:col10/1731456852750/Put/seqid=0 2024-11-13T00:14:13,909 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/bc5ba7f7129b4c61a09d17086792ead0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bc5ba7f7129b4c61a09d17086792ead0 2024-11-13T00:14:13,915 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into bc5ba7f7129b4c61a09d17086792ead0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:13,915 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:13,915 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456853389; duration=0sec 2024-11-13T00:14:13,915 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:13,915 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:13,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742039_1215 (size=31869) 2024-11-13T00:14:13,939 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/b807eb990f8f454ba7def0c69885586a 2024-11-13T00:14:13,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/699eeba2448d45938c709fea57b4ef37 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37 2024-11-13T00:14:13,949 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37, entries=150, sequenceid=326, filesize=30.5 K 2024-11-13T00:14:13,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/473c92f09ac44630b975937be628f435 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/473c92f09ac44630b975937be628f435 2024-11-13T00:14:13,957 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/473c92f09ac44630b975937be628f435, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T00:14:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/b807eb990f8f454ba7def0c69885586a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/b807eb990f8f454ba7def0c69885586a 2024-11-13T00:14:13,967 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/b807eb990f8f454ba7def0c69885586a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T00:14:13,968 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1ee97d494479a45762cab6f40ed6001d in 560ms, sequenceid=326, compaction requested=false 2024-11-13T00:14:13,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:13,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:13,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-13T00:14:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-13T00:14:13,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T00:14:13,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-13T00:14:13,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8120 sec 2024-11-13T00:14:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:14,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:14:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:14,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132f5d373ab3164de693f690ca5b087794_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:14,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742040_1216 (size=14994) 2024-11-13T00:14:14,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456914124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456914196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:14,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T00:14:14,221 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T00:14:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456914235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T00:14:14,327 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:14,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-13T00:14:14,328 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/ec27e49b63cc4437a26656100a2f3026 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ec27e49b63cc4437a26656100a2f3026 2024-11-13T00:14:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:14,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:14,330 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:14,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:14,337 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into ec27e49b63cc4437a26656100a2f3026(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:14,337 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:14,337 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456853389; duration=0sec 2024-11-13T00:14:14,337 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:14,337 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456914445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:14,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:14,499 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132f5d373ab3164de693f690ca5b087794_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f5d373ab3164de693f690ca5b087794_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:14,500 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a85b994c51b5472283a2a7bfe94b95f5, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:14,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a85b994c51b5472283a2a7bfe94b95f5 is 175, key is test_row_0/A:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:14,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742041_1217 (size=39949) 2024-11-13T00:14:14,508 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a85b994c51b5472283a2a7bfe94b95f5 2024-11-13T00:14:14,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/d2f93e017d3d4231907961b92b7792c1 is 50, key is test_row_0/B:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:14,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742042_1218 (size=12301) 2024-11-13T00:14:14,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:14,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456914706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:14,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456914756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:14,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/d2f93e017d3d4231907961b92b7792c1 2024-11-13T00:14:14,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/7b358a50b12e4d609bb39b915513634d is 50, key is test_row_0/C:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:14,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742043_1219 (size=12301) 2024-11-13T00:14:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:14,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:14,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:14,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:14,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:15,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:15,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456915266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:15,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:15,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/7b358a50b12e4d609bb39b915513634d 2024-11-13T00:14:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a85b994c51b5472283a2a7bfe94b95f5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5 2024-11-13T00:14:15,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5, entries=200, sequenceid=348, filesize=39.0 K 2024-11-13T00:14:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/d2f93e017d3d4231907961b92b7792c1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d2f93e017d3d4231907961b92b7792c1 2024-11-13T00:14:15,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d2f93e017d3d4231907961b92b7792c1, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T00:14:15,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/7b358a50b12e4d609bb39b915513634d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7b358a50b12e4d609bb39b915513634d 2024-11-13T00:14:15,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7b358a50b12e4d609bb39b915513634d, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T00:14:15,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1ee97d494479a45762cab6f40ed6001d in 1291ms, sequenceid=348, compaction requested=true 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:15,360 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:15,360 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:15,361 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:15,361 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:15,361 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:15,361 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:15,361 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,362 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,362 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ec27e49b63cc4437a26656100a2f3026, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=100.7 K 2024-11-13T00:14:15,362 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/33dbe666d2cd4a67bcf3bae376d7c39a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/473c92f09ac44630b975937be628f435, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d2f93e017d3d4231907961b92b7792c1] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.6 K 2024-11-13T00:14:15,362 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,362 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ec27e49b63cc4437a26656100a2f3026, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5] 2024-11-13T00:14:15,362 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 33dbe666d2cd4a67bcf3bae376d7c39a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731456852021 2024-11-13T00:14:15,362 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec27e49b63cc4437a26656100a2f3026, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731456852021 2024-11-13T00:14:15,363 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 473c92f09ac44630b975937be628f435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731456852796 2024-11-13T00:14:15,363 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 699eeba2448d45938c709fea57b4ef37, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731456852796 2024-11-13T00:14:15,364 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f93e017d3d4231907961b92b7792c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731456853534 2024-11-13T00:14:15,364 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85b994c51b5472283a2a7bfe94b95f5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731456853534 2024-11-13T00:14:15,370 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:15,373 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ab0f0c55d1a84f7585af4cdb7be0b446_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:15,373 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:15,374 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3fc2ed4ead6541778b1aa5f7d2c45f4c is 50, key is test_row_0/B:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:15,374 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ab0f0c55d1a84f7585af4cdb7be0b446_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:15,375 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ab0f0c55d1a84f7585af4cdb7be0b446_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:15,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742044_1220 (size=13017) 2024-11-13T00:14:15,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742045_1221 (size=4469) 2024-11-13T00:14:15,418 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3fc2ed4ead6541778b1aa5f7d2c45f4c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3fc2ed4ead6541778b1aa5f7d2c45f4c 2024-11-13T00:14:15,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T00:14:15,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,424 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:14:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:15,425 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#183 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:15,426 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/57ddf4103a2a4b238355208519c36865 is 175, key is test_row_0/A:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:15,428 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into 3fc2ed4ead6541778b1aa5f7d2c45f4c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:15,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:15,428 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456855360; duration=0sec 2024-11-13T00:14:15,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:15,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:15,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:15,433 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:15,434 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:15,434 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:15,434 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bc5ba7f7129b4c61a09d17086792ead0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/b807eb990f8f454ba7def0c69885586a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7b358a50b12e4d609bb39b915513634d] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.6 K 2024-11-13T00:14:15,434 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5ba7f7129b4c61a09d17086792ead0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731456852021 2024-11-13T00:14:15,436 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b807eb990f8f454ba7def0c69885586a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731456852796 2024-11-13T00:14:15,436 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b358a50b12e4d609bb39b915513634d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731456853534 2024-11-13T00:14:15,448 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:15,452 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/4d8a457cdeb94f9e8a4dedc9dc65b99d is 50, key is test_row_0/C:col10/1731456854066/Put/seqid=0 2024-11-13T00:14:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136ef354ba5ac9447396aa8e0e9cb6ba69_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456854122/Put/seqid=0 2024-11-13T00:14:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:15,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742046_1222 (size=31971) 2024-11-13T00:14:15,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742047_1223 (size=13017) 2024-11-13T00:14:15,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742048_1224 (size=12454) 2024-11-13T00:14:15,484 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/57ddf4103a2a4b238355208519c36865 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/57ddf4103a2a4b238355208519c36865 2024-11-13T00:14:15,490 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into 57ddf4103a2a4b238355208519c36865(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:15,490 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:15,490 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456855360; duration=0sec 2024-11-13T00:14:15,490 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:15,490 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:15,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456915845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:15,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136ef354ba5ac9447396aa8e0e9cb6ba69_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ef354ba5ac9447396aa8e0e9cb6ba69_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8efb9217b0f942df8b44c8d2783b0245, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8efb9217b0f942df8b44c8d2783b0245 is 175, key is test_row_0/A:col10/1731456854122/Put/seqid=0 2024-11-13T00:14:15,900 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/4d8a457cdeb94f9e8a4dedc9dc65b99d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d8a457cdeb94f9e8a4dedc9dc65b99d 2024-11-13T00:14:15,909 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 4d8a457cdeb94f9e8a4dedc9dc65b99d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:15,909 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:15,909 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456855360; duration=0sec 2024-11-13T00:14:15,909 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:15,909 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:15,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742049_1225 (size=31255) 2024-11-13T00:14:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456915956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456916166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:16,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456916286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:16,287 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:16,319 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8efb9217b0f942df8b44c8d2783b0245 2024-11-13T00:14:16,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a0d5a94b2d554f0dad0e870d6d96706d is 50, key is test_row_0/B:col10/1731456854122/Put/seqid=0 2024-11-13T00:14:16,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742050_1226 (size=12301) 2024-11-13T00:14:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456916476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:16,767 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a0d5a94b2d554f0dad0e870d6d96706d 2024-11-13T00:14:16,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/965f9ddf524a438fb4bdbbc8c51c08da is 50, key is test_row_0/C:col10/1731456854122/Put/seqid=0 2024-11-13T00:14:16,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742051_1227 (size=12301) 2024-11-13T00:14:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456916987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:17,202 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/965f9ddf524a438fb4bdbbc8c51c08da 2024-11-13T00:14:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8efb9217b0f942df8b44c8d2783b0245 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245 2024-11-13T00:14:17,212 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245, entries=150, sequenceid=361, filesize=30.5 K 2024-11-13T00:14:17,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/a0d5a94b2d554f0dad0e870d6d96706d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a0d5a94b2d554f0dad0e870d6d96706d 2024-11-13T00:14:17,218 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a0d5a94b2d554f0dad0e870d6d96706d, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T00:14:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/965f9ddf524a438fb4bdbbc8c51c08da as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/965f9ddf524a438fb4bdbbc8c51c08da 2024-11-13T00:14:17,226 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/965f9ddf524a438fb4bdbbc8c51c08da, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T00:14:17,227 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1ee97d494479a45762cab6f40ed6001d in 1803ms, sequenceid=361, compaction requested=false 2024-11-13T00:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T00:14:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T00:14:17,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T00:14:17,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8990 sec 2024-11-13T00:14:17,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9060 sec 2024-11-13T00:14:18,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456855758, max timestamp=1731456858012Num uniques 859; 2024-11-13T00:14:18,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:18,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456855758, max timestamp=1731456858012Num uniques 859; 2024-11-13T00:14:18,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:18,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-13T00:14:18,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:18,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:18,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132a1fc0d7d88f4a58aa53e5fa56738de2_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:18,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742052_1228 (size=14994) 2024-11-13T00:14:18,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:18,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132a1fc0d7d88f4a58aa53e5fa56738de2_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132a1fc0d7d88f4a58aa53e5fa56738de2_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:18,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/76830db3d7ab4c4982c02ef069e917e2, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:18,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/76830db3d7ab4c4982c02ef069e917e2 is 175, key is test_row_0/A:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:18,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742053_1229 (size=39949) 2024-11-13T00:14:18,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456858096, max timestamp=1731456858153Num uniques 859; 2024-11-13T00:14:18,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:18,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456858096, max timestamp=1731456858153Num uniques 859; 2024-11-13T00:14:18,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:18,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456858096, max timestamp=1731456858153Num uniques 859; 2024-11-13T00:14:18,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456918246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456918296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:18,300 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:18,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456918356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T00:14:18,497 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:18,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T00:14:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T00:14:18,500 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:18,501 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:18,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:18,531 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/76830db3d7ab4c4982c02ef069e917e2 2024-11-13T00:14:18,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3368dec63f9d4073a99b8950f4416674 is 50, key is test_row_0/B:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:18,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742054_1230 (size=12301) 2024-11-13T00:14:18,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3368dec63f9d4073a99b8950f4416674 2024-11-13T00:14:18,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/738753b932d54e6694be4d701d571aec is 50, key is test_row_0/C:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:18,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742055_1231 (size=12301) 2024-11-13T00:14:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456918566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T00:14:18,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T00:14:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T00:14:18,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:18,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T00:14:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456918876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:18,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T00:14:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:18,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/738753b932d54e6694be4d701d571aec 2024-11-13T00:14:18,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/76830db3d7ab4c4982c02ef069e917e2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2 2024-11-13T00:14:18,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2, entries=200, sequenceid=385, filesize=39.0 K 2024-11-13T00:14:18,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3368dec63f9d4073a99b8950f4416674 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3368dec63f9d4073a99b8950f4416674 2024-11-13T00:14:18,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3368dec63f9d4073a99b8950f4416674, entries=150, sequenceid=385, filesize=12.0 K 2024-11-13T00:14:18,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/738753b932d54e6694be4d701d571aec as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/738753b932d54e6694be4d701d571aec 2024-11-13T00:14:18,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/738753b932d54e6694be4d701d571aec, entries=150, sequenceid=385, filesize=12.0 K 2024-11-13T00:14:18,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 1ee97d494479a45762cab6f40ed6001d in 970ms, sequenceid=385, compaction requested=true 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:18,990 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:18,990 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:18,991 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:18,991 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:18,992 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,992 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/57ddf4103a2a4b238355208519c36865, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=100.8 K 2024-11-13T00:14:18,992 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,992 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/57ddf4103a2a4b238355208519c36865, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2] 2024-11-13T00:14:18,993 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ddf4103a2a4b238355208519c36865, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731456853534 2024-11-13T00:14:18,993 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efb9217b0f942df8b44c8d2783b0245, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731456854094 2024-11-13T00:14:18,994 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76830db3d7ab4c4982c02ef069e917e2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731456855758 2024-11-13T00:14:18,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:18,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:18,994 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:18,994 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3fc2ed4ead6541778b1aa5f7d2c45f4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a0d5a94b2d554f0dad0e870d6d96706d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3368dec63f9d4073a99b8950f4416674] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.7 K 2024-11-13T00:14:18,995 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc2ed4ead6541778b1aa5f7d2c45f4c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731456853534 2024-11-13T00:14:18,995 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d5a94b2d554f0dad0e870d6d96706d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731456854094 2024-11-13T00:14:18,996 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3368dec63f9d4073a99b8950f4416674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731456855832 2024-11-13T00:14:19,004 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:19,007 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:19,007 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/fa2c39ab92c64228910afe3d11a35d68 is 50, key is test_row_0/B:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:19,011 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c96beb923aaf4d81b9bf15737510295a_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:19,013 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c96beb923aaf4d81b9bf15737510295a_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:19,013 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c96beb923aaf4d81b9bf15737510295a_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:19,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742056_1232 (size=13119) 2024-11-13T00:14:19,033 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/fa2c39ab92c64228910afe3d11a35d68 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fa2c39ab92c64228910afe3d11a35d68 2024-11-13T00:14:19,042 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into fa2c39ab92c64228910afe3d11a35d68(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:19,042 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:19,042 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456858990; duration=0sec 2024-11-13T00:14:19,042 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:19,043 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:19,043 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:19,044 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:19,044 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:19,044 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:19,044 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d8a457cdeb94f9e8a4dedc9dc65b99d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/965f9ddf524a438fb4bdbbc8c51c08da, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/738753b932d54e6694be4d701d571aec] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.7 K 2024-11-13T00:14:19,045 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8a457cdeb94f9e8a4dedc9dc65b99d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731456853534 2024-11-13T00:14:19,045 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 965f9ddf524a438fb4bdbbc8c51c08da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731456854094 2024-11-13T00:14:19,046 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 738753b932d54e6694be4d701d571aec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731456855832 2024-11-13T00:14:19,059 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:19,060 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/3799a4ba9b084743a7b08b057632afc0 is 50, key is test_row_0/C:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:19,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742057_1233 (size=4469) 2024-11-13T00:14:19,065 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#192 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:19,066 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/df2d5e66d2a84c75b5dd3a2ae558f9c6 is 175, key is test_row_0/A:col10/1731456855842/Put/seqid=0 2024-11-13T00:14:19,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T00:14:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:19,118 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T00:14:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:19,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742058_1234 (size=13119) 2024-11-13T00:14:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742059_1235 (size=32073) 2024-11-13T00:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T00:14:19,131 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/df2d5e66d2a84c75b5dd3a2ae558f9c6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/df2d5e66d2a84c75b5dd3a2ae558f9c6 2024-11-13T00:14:19,132 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/3799a4ba9b084743a7b08b057632afc0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3799a4ba9b084743a7b08b057632afc0 2024-11-13T00:14:19,147 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into df2d5e66d2a84c75b5dd3a2ae558f9c6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:19,147 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:19,147 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456858990; duration=0sec 2024-11-13T00:14:19,147 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:19,147 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:19,151 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 3799a4ba9b084743a7b08b057632afc0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:19,151 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:19,151 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456858990; duration=0sec 2024-11-13T00:14:19,151 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:19,151 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111365b6dbb838034fd3ae2c74142eacfb0a_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456858243/Put/seqid=0 2024-11-13T00:14:19,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742060_1236 (size=12454) 2024-11-13T00:14:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:19,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:19,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456919431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456919546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:19,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:19,598 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111365b6dbb838034fd3ae2c74142eacfb0a_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111365b6dbb838034fd3ae2c74142eacfb0a_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:19,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1f35c3ae6a554bcfae665451ed904196, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:19,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1f35c3ae6a554bcfae665451ed904196 is 175, key is test_row_0/A:col10/1731456858243/Put/seqid=0 2024-11-13T00:14:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742061_1237 (size=31255) 2024-11-13T00:14:19,610 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1f35c3ae6a554bcfae665451ed904196 2024-11-13T00:14:19,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d is 50, key is test_row_0/B:col10/1731456858243/Put/seqid=0 2024-11-13T00:14:19,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742062_1238 (size=12301) 2024-11-13T00:14:19,632 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d 2024-11-13T00:14:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T00:14:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/3e8be7b2f1dd43468b1c94b07fda5f8e is 50, key is test_row_0/C:col10/1731456858243/Put/seqid=0 2024-11-13T00:14:19,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742063_1239 (size=12301) 2024-11-13T00:14:19,671 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/3e8be7b2f1dd43468b1c94b07fda5f8e 2024-11-13T00:14:19,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/1f35c3ae6a554bcfae665451ed904196 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196 2024-11-13T00:14:19,686 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196, entries=150, sequenceid=421, filesize=30.5 K 2024-11-13T00:14:19,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d 2024-11-13T00:14:19,693 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d, entries=150, sequenceid=421, filesize=12.0 K 2024-11-13T00:14:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/3e8be7b2f1dd43468b1c94b07fda5f8e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3e8be7b2f1dd43468b1c94b07fda5f8e 2024-11-13T00:14:19,701 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3e8be7b2f1dd43468b1c94b07fda5f8e, entries=150, sequenceid=421, filesize=12.0 K 2024-11-13T00:14:19,706 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1ee97d494479a45762cab6f40ed6001d in 588ms, sequenceid=421, compaction requested=false 2024-11-13T00:14:19,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:19,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:19,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T00:14:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T00:14:19,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T00:14:19,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2090 sec 2024-11-13T00:14:19,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.2160 sec 2024-11-13T00:14:19,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:14:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:19,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113817c397cdfbf4e2abcc4311ce49cad4b_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:19,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742064_1240 (size=14994) 2024-11-13T00:14:19,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456919879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:19,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456919986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:20,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456920195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:20,202 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:20,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113817c397cdfbf4e2abcc4311ce49cad4b_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113817c397cdfbf4e2abcc4311ce49cad4b_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:20,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3bf082f39f58466989eb97295a7c4586, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:20,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3bf082f39f58466989eb97295a7c4586 is 175, key is test_row_0/A:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742065_1241 (size=39949) 2024-11-13T00:14:20,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3bf082f39f58466989eb97295a7c4586 2024-11-13T00:14:20,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3e394f1b06004658bc77dcd2d80aa692 is 50, key is test_row_0/B:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:20,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742066_1242 (size=12301) 2024-11-13T00:14:20,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3e394f1b06004658bc77dcd2d80aa692 2024-11-13T00:14:20,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/1cf5ff7f18294b6db912ea9de7f47e27 is 50, key is test_row_0/C:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742067_1243 (size=12301) 2024-11-13T00:14:20,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/1cf5ff7f18294b6db912ea9de7f47e27 2024-11-13T00:14:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3bf082f39f58466989eb97295a7c4586 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586 2024-11-13T00:14:20,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586, entries=200, sequenceid=435, filesize=39.0 K 2024-11-13T00:14:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/3e394f1b06004658bc77dcd2d80aa692 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3e394f1b06004658bc77dcd2d80aa692 2024-11-13T00:14:20,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3e394f1b06004658bc77dcd2d80aa692, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T00:14:20,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/1cf5ff7f18294b6db912ea9de7f47e27 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1cf5ff7f18294b6db912ea9de7f47e27 2024-11-13T00:14:20,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1cf5ff7f18294b6db912ea9de7f47e27, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T00:14:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1ee97d494479a45762cab6f40ed6001d in 531ms, sequenceid=435, compaction requested=true 2024-11-13T00:14:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:20,289 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:20,290 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:20,291 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:20,291 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:20,291 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:20,291 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:20,291 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,291 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fa2c39ab92c64228910afe3d11a35d68, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3e394f1b06004658bc77dcd2d80aa692] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.8 K 2024-11-13T00:14:20,291 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,291 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/df2d5e66d2a84c75b5dd3a2ae558f9c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=100.9 K 2024-11-13T00:14:20,291 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,292 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/df2d5e66d2a84c75b5dd3a2ae558f9c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586] 2024-11-13T00:14:20,292 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2c39ab92c64228910afe3d11a35d68, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731456855832 2024-11-13T00:14:20,293 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting df2d5e66d2a84c75b5dd3a2ae558f9c6, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731456855832 2024-11-13T00:14:20,293 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9ebdb6107f4dbc8c50bfbd3dd2825d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731456858212 2024-11-13T00:14:20,293 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f35c3ae6a554bcfae665451ed904196, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731456858212 2024-11-13T00:14:20,293 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e394f1b06004658bc77dcd2d80aa692, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731456859418 2024-11-13T00:14:20,294 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf082f39f58466989eb97295a7c4586, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731456859418 2024-11-13T00:14:20,321 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:20,322 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/ddaf7c14510e4554bf4dfc561b577e09 is 50, key is test_row_0/B:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:20,325 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:20,331 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111344712ef798aa4fbaa58f5e720ad41ad0_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:20,334 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111344712ef798aa4fbaa58f5e720ad41ad0_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:20,334 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111344712ef798aa4fbaa58f5e720ad41ad0_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:20,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742068_1244 (size=13221) 2024-11-13T00:14:20,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742069_1245 (size=4469) 2024-11-13T00:14:20,341 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#202 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:20,342 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/e6061cd3819c4b8ea52d76dde3f0e8d6 is 175, key is test_row_0/A:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:20,348 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/ddaf7c14510e4554bf4dfc561b577e09 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ddaf7c14510e4554bf4dfc561b577e09 2024-11-13T00:14:20,358 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into ddaf7c14510e4554bf4dfc561b577e09(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:20,358 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:20,358 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456860289; duration=0sec 2024-11-13T00:14:20,358 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:20,358 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:20,358 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:20,361 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:20,361 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:20,361 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,362 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3799a4ba9b084743a7b08b057632afc0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3e8be7b2f1dd43468b1c94b07fda5f8e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1cf5ff7f18294b6db912ea9de7f47e27] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=36.8 K 2024-11-13T00:14:20,362 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3799a4ba9b084743a7b08b057632afc0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731456855832 2024-11-13T00:14:20,363 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e8be7b2f1dd43468b1c94b07fda5f8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731456858212 2024-11-13T00:14:20,363 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf5ff7f18294b6db912ea9de7f47e27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731456859418 2024-11-13T00:14:20,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742070_1246 (size=32175) 2024-11-13T00:14:20,373 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:20,373 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/7a3bbafbc0904431b43876d94052b44a is 50, key is test_row_0/C:col10/1731456859429/Put/seqid=0 2024-11-13T00:14:20,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742071_1247 (size=13221) 2024-11-13T00:14:20,394 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/7a3bbafbc0904431b43876d94052b44a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7a3bbafbc0904431b43876d94052b44a 2024-11-13T00:14:20,402 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 7a3bbafbc0904431b43876d94052b44a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:20,402 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:20,402 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456860290; duration=0sec 2024-11-13T00:14:20,403 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:20,403 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:20,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456859864, max timestamp=1731456860507Num uniques 859; 2024-11-13T00:14:20,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:20,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456859864, max timestamp=1731456860507Num uniques 859; 2024-11-13T00:14:20,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:20,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-13T00:14:20,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:20,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:20,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:20,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:20,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:20,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:20,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113721b3b5050094c10b971f3b5b2c559cc_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456859873/Put/seqid=0 2024-11-13T00:14:20,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742072_1248 (size=14994) 2024-11-13T00:14:20,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456860564, max timestamp=1731456860584Num uniques 859; 2024-11-13T00:14:20,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:20,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456860564, max timestamp=1731456860584Num uniques 859; 2024-11-13T00:14:20,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:20,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456860564, max timestamp=1731456860584Num uniques 859; 2024-11-13T00:14:20,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T00:14:20,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:20,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T00:14:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T00:14:20,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:20,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:20,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:20,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456920652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T00:14:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456920766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:20,779 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/e6061cd3819c4b8ea52d76dde3f0e8d6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/e6061cd3819c4b8ea52d76dde3f0e8d6 2024-11-13T00:14:20,784 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into e6061cd3819c4b8ea52d76dde3f0e8d6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:20,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:20,784 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456860289; duration=0sec 2024-11-13T00:14:20,785 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:20,785 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:20,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:20,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:20,939 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:20,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113721b3b5050094c10b971f3b5b2c559cc_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113721b3b5050094c10b971f3b5b2c559cc_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:20,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/34b294c4941f4e4c935549f119f573c7, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:20,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/34b294c4941f4e4c935549f119f573c7 is 175, key is test_row_0/A:col10/1731456859873/Put/seqid=0 2024-11-13T00:14:20,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742073_1249 (size=39949) 2024-11-13T00:14:20,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:20,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T00:14:20,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456920975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:21,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T00:14:21,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456921286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:21,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/34b294c4941f4e4c935549f119f573c7 2024-11-13T00:14:21,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/cca76ef348834aba913edb2a68672bb8 is 50, key is test_row_0/B:col10/1731456859873/Put/seqid=0 2024-11-13T00:14:21,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742074_1250 (size=12301) 2024-11-13T00:14:21,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/cca76ef348834aba913edb2a68672bb8 2024-11-13T00:14:21,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/ed3f536635634611b2e778e18392aae8 is 50, key is test_row_0/C:col10/1731456859873/Put/seqid=0 2024-11-13T00:14:21,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742075_1251 (size=12301) 2024-11-13T00:14:21,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:21,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T00:14:21,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456921796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:21,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/ed3f536635634611b2e778e18392aae8 2024-11-13T00:14:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/34b294c4941f4e4c935549f119f573c7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7 2024-11-13T00:14:21,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7, entries=200, sequenceid=458, filesize=39.0 K 2024-11-13T00:14:21,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/cca76ef348834aba913edb2a68672bb8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/cca76ef348834aba913edb2a68672bb8 2024-11-13T00:14:21,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/cca76ef348834aba913edb2a68672bb8, entries=150, sequenceid=458, filesize=12.0 K 2024-11-13T00:14:21,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/ed3f536635634611b2e778e18392aae8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ed3f536635634611b2e778e18392aae8 2024-11-13T00:14:21,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ed3f536635634611b2e778e18392aae8, entries=150, sequenceid=458, filesize=12.0 K 2024-11-13T00:14:21,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 1ee97d494479a45762cab6f40ed6001d in 1323ms, sequenceid=458, compaction requested=false 2024-11-13T00:14:21,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:21,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T00:14:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:21,880 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T00:14:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:21,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137de709a1af2249cfa024aa966a3a99fe_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456860643/Put/seqid=0 2024-11-13T00:14:21,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742076_1252 (size=12454) 2024-11-13T00:14:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:22,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:22,341 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137de709a1af2249cfa024aa966a3a99fe_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137de709a1af2249cfa024aa966a3a99fe_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:22,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3d8ec131019f4c6cb882ca2aa6655239, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:22,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3d8ec131019f4c6cb882ca2aa6655239 is 175, key is test_row_0/A:col10/1731456860643/Put/seqid=0 2024-11-13T00:14:22,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742077_1253 (size=31255) 2024-11-13T00:14:22,357 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3d8ec131019f4c6cb882ca2aa6655239 2024-11-13T00:14:22,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/840e17aeb20b42958d1afe2917f99be8 is 50, key is test_row_0/B:col10/1731456860643/Put/seqid=0 2024-11-13T00:14:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456922365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:22,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742078_1254 (size=12301) 2024-11-13T00:14:22,383 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/840e17aeb20b42958d1afe2917f99be8 2024-11-13T00:14:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/5702066677e84910ba9a68c87442e776 is 50, key is test_row_0/C:col10/1731456860643/Put/seqid=0 2024-11-13T00:14:22,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742079_1255 (size=12301) 2024-11-13T00:14:22,423 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/5702066677e84910ba9a68c87442e776 2024-11-13T00:14:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/3d8ec131019f4c6cb882ca2aa6655239 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239 2024-11-13T00:14:22,435 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239, entries=150, sequenceid=494, filesize=30.5 K 2024-11-13T00:14:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/840e17aeb20b42958d1afe2917f99be8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/840e17aeb20b42958d1afe2917f99be8 2024-11-13T00:14:22,440 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/840e17aeb20b42958d1afe2917f99be8, entries=150, sequenceid=494, filesize=12.0 K 2024-11-13T00:14:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/5702066677e84910ba9a68c87442e776 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/5702066677e84910ba9a68c87442e776 2024-11-13T00:14:22,446 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/5702066677e84910ba9a68c87442e776, entries=150, sequenceid=494, filesize=12.0 K 2024-11-13T00:14:22,447 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1ee97d494479a45762cab6f40ed6001d in 567ms, sequenceid=494, compaction requested=true 2024-11-13T00:14:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T00:14:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T00:14:22,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T00:14:22,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-13T00:14:22,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.8050 sec 2024-11-13T00:14:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:22,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:14:22,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:22,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:22,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130888ce3ad9ae494eb30b5e5dd3ea5d2f_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:22,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742080_1256 (size=12454) 2024-11-13T00:14:22,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456922611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:22,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456922716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:22,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T00:14:22,796 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:22,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T00:14:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T00:14:22,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:22,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:22,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456922816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:22,817 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:22,822 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:14:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T00:14:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456922925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:22,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T00:14:22,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:22,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:22,963 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:22,968 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130888ce3ad9ae494eb30b5e5dd3ea5d2f_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130888ce3ad9ae494eb30b5e5dd3ea5d2f_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:22,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a39335c2046f48dab40515fc66e8cc40, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:22,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a39335c2046f48dab40515fc66e8cc40 is 175, key is test_row_0/A:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:22,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742081_1257 (size=31255) 2024-11-13T00:14:22,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=506, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a39335c2046f48dab40515fc66e8cc40 2024-11-13T00:14:22,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/ca33b736fc524416954b5b3e9d24fa34 is 50, key is test_row_0/B:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:22,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742082_1258 (size=12301) 2024-11-13T00:14:23,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T00:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T00:14:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456923236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T00:14:23,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/ca33b736fc524416954b5b3e9d24fa34 2024-11-13T00:14:23,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T00:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T00:14:23,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/742beccf67934ac78f61121d51f89d93 is 50, key is test_row_0/C:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:23,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742083_1259 (size=12301) 2024-11-13T00:14:23,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/742beccf67934ac78f61121d51f89d93 2024-11-13T00:14:23,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/a39335c2046f48dab40515fc66e8cc40 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40 2024-11-13T00:14:23,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40, entries=150, sequenceid=506, filesize=30.5 K 2024-11-13T00:14:23,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/ca33b736fc524416954b5b3e9d24fa34 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ca33b736fc524416954b5b3e9d24fa34 2024-11-13T00:14:23,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ca33b736fc524416954b5b3e9d24fa34, entries=150, sequenceid=506, filesize=12.0 K 2024-11-13T00:14:23,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/742beccf67934ac78f61121d51f89d93 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/742beccf67934ac78f61121d51f89d93 2024-11-13T00:14:23,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/742beccf67934ac78f61121d51f89d93, entries=150, sequenceid=506, filesize=12.0 K 2024-11-13T00:14:23,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1ee97d494479a45762cab6f40ed6001d in 1055ms, sequenceid=506, compaction requested=true 2024-11-13T00:14:23,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:23,534 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:23,534 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:23,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:23,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:23,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:23,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:23,537 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:23,538 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:23,538 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,538 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/e6061cd3819c4b8ea52d76dde3f0e8d6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=131.5 K 2024-11-13T00:14:23,538 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,538 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/e6061cd3819c4b8ea52d76dde3f0e8d6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40] 2024-11-13T00:14:23,539 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6061cd3819c4b8ea52d76dde3f0e8d6, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731456859418 2024-11-13T00:14:23,539 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:23,539 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:23,539 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,539 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ddaf7c14510e4554bf4dfc561b577e09, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/cca76ef348834aba913edb2a68672bb8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/840e17aeb20b42958d1afe2917f99be8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ca33b736fc524416954b5b3e9d24fa34] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=48.9 K 2024-11-13T00:14:23,540 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b294c4941f4e4c935549f119f573c7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731456859864 2024-11-13T00:14:23,540 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ddaf7c14510e4554bf4dfc561b577e09, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731456859418 2024-11-13T00:14:23,540 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d8ec131019f4c6cb882ca2aa6655239, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731456860633 2024-11-13T00:14:23,541 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting cca76ef348834aba913edb2a68672bb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731456859864 2024-11-13T00:14:23,541 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39335c2046f48dab40515fc66e8cc40, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731456862346 2024-11-13T00:14:23,542 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 840e17aeb20b42958d1afe2917f99be8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731456860633 2024-11-13T00:14:23,542 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ca33b736fc524416954b5b3e9d24fa34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731456862346 2024-11-13T00:14:23,557 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:23,560 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111309cdb6d24a74464892ac946531b246e3_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:23,562 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:23,562 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111309cdb6d24a74464892ac946531b246e3_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:23,563 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111309cdb6d24a74464892ac946531b246e3_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:23,563 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5f5a39e9e2cb485488d29373cf2aaa46 is 50, key is test_row_0/B:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:23,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T00:14:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,568 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:14:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:23,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d8777b5675c04cc2be82e76c15aea8b8_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456862604/Put/seqid=0 2024-11-13T00:14:23,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742085_1261 (size=13357) 2024-11-13T00:14:23,602 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5f5a39e9e2cb485488d29373cf2aaa46 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5f5a39e9e2cb485488d29373cf2aaa46 2024-11-13T00:14:23,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742084_1260 (size=4469) 2024-11-13T00:14:23,608 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into 5f5a39e9e2cb485488d29373cf2aaa46(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:23,608 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:23,608 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=12, startTime=1731456863534; duration=0sec 2024-11-13T00:14:23,608 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:23,608 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:23,609 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:23,610 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:23,610 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:23,610 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:23,610 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7a3bbafbc0904431b43876d94052b44a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ed3f536635634611b2e778e18392aae8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/5702066677e84910ba9a68c87442e776, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/742beccf67934ac78f61121d51f89d93] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=48.9 K 2024-11-13T00:14:23,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742086_1262 (size=12454) 2024-11-13T00:14:23,611 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3bbafbc0904431b43876d94052b44a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731456859418 2024-11-13T00:14:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:23,611 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3f536635634611b2e778e18392aae8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731456859864 2024-11-13T00:14:23,612 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5702066677e84910ba9a68c87442e776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731456860633 2024-11-13T00:14:23,612 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 742beccf67934ac78f61121d51f89d93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731456862346 2024-11-13T00:14:23,617 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d8777b5675c04cc2be82e76c15aea8b8_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d8777b5675c04cc2be82e76c15aea8b8_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/575643a335064e45ac5a251e080b61c6, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:23,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/575643a335064e45ac5a251e080b61c6 is 175, key is test_row_0/A:col10/1731456862604/Put/seqid=0 2024-11-13T00:14:23,627 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:23,628 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/f849b96dd37b4c3bb05b3ac5f0e7cf99 is 50, key is test_row_0/C:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:23,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742087_1263 (size=31255) 2024-11-13T00:14:23,654 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=526, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/575643a335064e45ac5a251e080b61c6 2024-11-13T00:14:23,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742088_1264 (size=13357) 2024-11-13T00:14:23,667 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/f849b96dd37b4c3bb05b3ac5f0e7cf99 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f849b96dd37b4c3bb05b3ac5f0e7cf99 2024-11-13T00:14:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/f8cc712f70f24c649df01d59c2faf873 is 50, key is test_row_0/B:col10/1731456862604/Put/seqid=0 2024-11-13T00:14:23,677 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into f849b96dd37b4c3bb05b3ac5f0e7cf99(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:23,677 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:23,677 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=12, startTime=1731456863535; duration=0sec 2024-11-13T00:14:23,677 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:23,677 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:23,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742089_1265 (size=12301) 2024-11-13T00:14:23,703 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/f8cc712f70f24c649df01d59c2faf873 2024-11-13T00:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. as already flushing 2024-11-13T00:14:23,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/f9b1d5e82a6b4d2885213e7c4e793d2c is 50, key is test_row_0/C:col10/1731456862604/Put/seqid=0 2024-11-13T00:14:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456923732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742090_1266 (size=12301) 2024-11-13T00:14:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456923739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456923746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49130 deadline: 1731456923816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,819 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18346 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456923846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456923856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T00:14:24,007 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#213 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:24,008 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/849df79e00c24fc4a6b376650171dcc6 is 175, key is test_row_0/A:col10/1731456862346/Put/seqid=0 2024-11-13T00:14:24,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742091_1267 (size=32311) 2024-11-13T00:14:24,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456924057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456924066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,140 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/f9b1d5e82a6b4d2885213e7c4e793d2c 2024-11-13T00:14:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/575643a335064e45ac5a251e080b61c6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6 2024-11-13T00:14:24,160 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6, entries=150, sequenceid=526, filesize=30.5 K 2024-11-13T00:14:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/f8cc712f70f24c649df01d59c2faf873 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f8cc712f70f24c649df01d59c2faf873 2024-11-13T00:14:24,171 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f8cc712f70f24c649df01d59c2faf873, entries=150, sequenceid=526, filesize=12.0 K 2024-11-13T00:14:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/f9b1d5e82a6b4d2885213e7c4e793d2c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f9b1d5e82a6b4d2885213e7c4e793d2c 2024-11-13T00:14:24,176 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f9b1d5e82a6b4d2885213e7c4e793d2c, entries=150, sequenceid=526, filesize=12.0 K 2024-11-13T00:14:24,178 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1ee97d494479a45762cab6f40ed6001d in 610ms, sequenceid=526, compaction requested=false 2024-11-13T00:14:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T00:14:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T00:14:24,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T00:14:24,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3800 sec 2024-11-13T00:14:24,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.3870 sec 2024-11-13T00:14:24,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:14:24,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:24,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:24,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:24,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:24,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:24,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:24,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133084cf257ef44b06bad00e2cf46df497_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742092_1268 (size=14994) 2024-11-13T00:14:24,424 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/849df79e00c24fc4a6b376650171dcc6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/849df79e00c24fc4a6b376650171dcc6 2024-11-13T00:14:24,432 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into 849df79e00c24fc4a6b376650171dcc6(size=31.6 K), total size for store is 62.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:24,432 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:24,433 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=12, startTime=1731456863534; duration=0sec 2024-11-13T00:14:24,433 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:24,433 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456924432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456924439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456924546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456924556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456924756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1731456924756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,762 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456924766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,774 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-13T00:14:24,774 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:24,774 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,774 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:24,778 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-13T00:14:24,778 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:24,778 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,779 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:24,779 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-13T00:14:24,779 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:24,779 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,779 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:24,782 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-13T00:14:24,782 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:24,782 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,782 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:24,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:24,820 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:24,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133084cf257ef44b06bad00e2cf46df497_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133084cf257ef44b06bad00e2cf46df497_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:24,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/2387794d4b654291923891cbbd62884a, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:24,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/2387794d4b654291923891cbbd62884a is 175, key is test_row_0/A:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:24,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49044 deadline: 1731456924836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:24,838 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 1ee97d494479a45762cab6f40ed6001d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:24,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742093_1269 (size=39949) 2024-11-13T00:14:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T00:14:24,957 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:25,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456925067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456925076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:25,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=541, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/2387794d4b654291923891cbbd62884a 2024-11-13T00:14:25,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06 is 50, key is test_row_0/B:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:25,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742094_1270 (size=12301) 2024-11-13T00:14:25,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49064 deadline: 1731456925576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49120 deadline: 1731456925586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:14:25,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06 2024-11-13T00:14:25,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/cd9afdd437164402a13f533c4f2d5660 is 50, key is test_row_0/C:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742095_1271 (size=12301) 2024-11-13T00:14:26,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/cd9afdd437164402a13f533c4f2d5660 2024-11-13T00:14:26,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/2387794d4b654291923891cbbd62884a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a 2024-11-13T00:14:26,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a, entries=200, sequenceid=541, filesize=39.0 K 2024-11-13T00:14:26,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06 2024-11-13T00:14:26,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06, entries=150, sequenceid=541, filesize=12.0 K 2024-11-13T00:14:26,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/cd9afdd437164402a13f533c4f2d5660 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cd9afdd437164402a13f533c4f2d5660 2024-11-13T00:14:26,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cd9afdd437164402a13f533c4f2d5660, entries=150, sequenceid=541, filesize=12.0 K 2024-11-13T00:14:26,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1ee97d494479a45762cab6f40ed6001d in 1710ms, sequenceid=541, compaction requested=true 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:26,081 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1ee97d494479a45762cab6f40ed6001d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:26,081 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:26,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:26,082 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:26,082 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:26,082 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/B is initiating minor compaction (all files) 2024-11-13T00:14:26,082 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/A is initiating minor compaction (all files) 2024-11-13T00:14:26,082 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/B in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:26,082 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/A in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:26,082 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5f5a39e9e2cb485488d29373cf2aaa46, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f8cc712f70f24c649df01d59c2faf873, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=37.1 K 2024-11-13T00:14:26,082 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/849df79e00c24fc4a6b376650171dcc6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=101.1 K 2024-11-13T00:14:26,082 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:26,082 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/849df79e00c24fc4a6b376650171dcc6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a] 2024-11-13T00:14:26,082 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5a39e9e2cb485488d29373cf2aaa46, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731456862346 2024-11-13T00:14:26,083 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 849df79e00c24fc4a6b376650171dcc6, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731456862346 2024-11-13T00:14:26,083 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f8cc712f70f24c649df01d59c2faf873, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1731456862525 2024-11-13T00:14:26,083 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b8bc29bb0cb4d6cbdc9d729ab25cf06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731456863729 2024-11-13T00:14:26,083 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575643a335064e45ac5a251e080b61c6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1731456862525 2024-11-13T00:14:26,083 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2387794d4b654291923891cbbd62884a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731456863715 2024-11-13T00:14:26,092 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:26,092 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/fd99456f4a334a15a4a9e457d2e56757 is 50, key is test_row_0/B:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:26,093 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:26,096 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ab1a80702e764ecaa63cc7acc5cb45fa_1ee97d494479a45762cab6f40ed6001d store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:26,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742096_1272 (size=13459) 2024-11-13T00:14:26,119 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ab1a80702e764ecaa63cc7acc5cb45fa_1ee97d494479a45762cab6f40ed6001d, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:26,120 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ab1a80702e764ecaa63cc7acc5cb45fa_1ee97d494479a45762cab6f40ed6001d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:26,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742097_1273 (size=4469) 2024-11-13T00:14:26,137 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#A#compaction#223 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:26,138 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8fb76c9072de4ea981afea49f4e024b5 is 175, key is test_row_0/A:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742098_1274 (size=32413) 2024-11-13T00:14:26,504 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/fd99456f4a334a15a4a9e457d2e56757 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fd99456f4a334a15a4a9e457d2e56757 2024-11-13T00:14:26,508 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/B of 1ee97d494479a45762cab6f40ed6001d into fd99456f4a334a15a4a9e457d2e56757(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:26,508 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:26,508 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/B, priority=13, startTime=1731456866081; duration=0sec 2024-11-13T00:14:26,508 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:26,508 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:B 2024-11-13T00:14:26,508 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:26,509 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:26,510 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 1ee97d494479a45762cab6f40ed6001d/C is initiating minor compaction (all files) 2024-11-13T00:14:26,510 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1ee97d494479a45762cab6f40ed6001d/C in TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:26,510 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f849b96dd37b4c3bb05b3ac5f0e7cf99, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f9b1d5e82a6b4d2885213e7c4e793d2c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cd9afdd437164402a13f533c4f2d5660] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp, totalSize=37.1 K 2024-11-13T00:14:26,510 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f849b96dd37b4c3bb05b3ac5f0e7cf99, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731456862346 2024-11-13T00:14:26,511 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b1d5e82a6b4d2885213e7c4e793d2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1731456862525 2024-11-13T00:14:26,511 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9afdd437164402a13f533c4f2d5660, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731456863729 2024-11-13T00:14:26,520 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1ee97d494479a45762cab6f40ed6001d#C#compaction#224 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:26,521 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/998195d6a3714c87aa8f7ed393d2632d is 50, key is test_row_0/C:col10/1731456863736/Put/seqid=0 2024-11-13T00:14:26,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742099_1275 (size=13459) 2024-11-13T00:14:26,529 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/998195d6a3714c87aa8f7ed393d2632d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/998195d6a3714c87aa8f7ed393d2632d 2024-11-13T00:14:26,534 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/C of 1ee97d494479a45762cab6f40ed6001d into 998195d6a3714c87aa8f7ed393d2632d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:26,534 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:26,534 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/C, priority=13, startTime=1731456866081; duration=0sec 2024-11-13T00:14:26,534 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:26,534 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:C 2024-11-13T00:14:26,559 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/8fb76c9072de4ea981afea49f4e024b5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fb76c9072de4ea981afea49f4e024b5 2024-11-13T00:14:26,564 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1ee97d494479a45762cab6f40ed6001d/A of 1ee97d494479a45762cab6f40ed6001d into 8fb76c9072de4ea981afea49f4e024b5(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:26,564 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:26,565 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., storeName=1ee97d494479a45762cab6f40ed6001d/A, priority=13, startTime=1731456866081; duration=0sec 2024-11-13T00:14:26,565 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:26,565 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1ee97d494479a45762cab6f40ed6001d:A 2024-11-13T00:14:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:26,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:14:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:26,588 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-13T00:14:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:26,589 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:26,589 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:26,589 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:26,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:26,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132646b9e9662340779003d971e0e9d2a6_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456864428/Put/seqid=0 2024-11-13T00:14:26,598 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-13T00:14:26,598 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:26,598 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:26,598 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:26,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:26,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742100_1276 (size=12454) 2024-11-13T00:14:26,768 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-13T00:14:26,768 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:26,768 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:26,768 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:26,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:27,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:27,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132646b9e9662340779003d971e0e9d2a6_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132646b9e9662340779003d971e0e9d2a6_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:27,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/411444d407cf4c0ca3f36a3daa7f1365, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:27,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/411444d407cf4c0ca3f36a3daa7f1365 is 175, key is test_row_0/A:col10/1731456864428/Put/seqid=0 2024-11-13T00:14:27,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742101_1277 (size=31255) 2024-11-13T00:14:27,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=565, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/411444d407cf4c0ca3f36a3daa7f1365 2024-11-13T00:14:27,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/569a947d5d774bc4a6baa1992b864844 is 50, key is test_row_0/B:col10/1731456864428/Put/seqid=0 2024-11-13T00:14:27,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742102_1278 (size=12301) 2024-11-13T00:14:27,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/569a947d5d774bc4a6baa1992b864844 2024-11-13T00:14:27,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/412e9b608f0941eeb24e4a4759c43bda is 50, key is test_row_0/C:col10/1731456864428/Put/seqid=0 2024-11-13T00:14:27,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742103_1279 (size=12301) 2024-11-13T00:14:28,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/412e9b608f0941eeb24e4a4759c43bda 2024-11-13T00:14:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/411444d407cf4c0ca3f36a3daa7f1365 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/411444d407cf4c0ca3f36a3daa7f1365 2024-11-13T00:14:28,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/411444d407cf4c0ca3f36a3daa7f1365, entries=150, sequenceid=565, filesize=30.5 K 2024-11-13T00:14:28,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/569a947d5d774bc4a6baa1992b864844 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/569a947d5d774bc4a6baa1992b864844 2024-11-13T00:14:28,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/569a947d5d774bc4a6baa1992b864844, entries=150, sequenceid=565, filesize=12.0 K 2024-11-13T00:14:28,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/412e9b608f0941eeb24e4a4759c43bda as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/412e9b608f0941eeb24e4a4759c43bda 2024-11-13T00:14:28,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/412e9b608f0941eeb24e4a4759c43bda, entries=150, sequenceid=565, filesize=12.0 K 2024-11-13T00:14:28,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 1ee97d494479a45762cab6f40ed6001d in 1672ms, sequenceid=565, compaction requested=false 2024-11-13T00:14:28,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1ee97d494479a45762cab6f40ed6001d: 2024-11-13T00:14:28,877 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-13T00:14:28,877 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:28,877 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:28,877 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:28,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:33,918 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-13T00:14:33,918 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:33,918 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:33,918 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:33,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 267 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 3 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2491 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2488 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1123 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3369 rows 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-13T00:14:33,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-11-13T00:14:33,918 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:14:33,919 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:14:33,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:33,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:33,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:33,919 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T00:14:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T00:14:33,922 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456873922"}]},"ts":"1731456873922"} 2024-11-13T00:14:33,923 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T00:14:33,923 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T00:14:33,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:14:33,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, UNASSIGN}] 2024-11-13T00:14:33,926 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, UNASSIGN 2024-11-13T00:14:33,926 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:33,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, UNASSIGN because future has completed 2024-11-13T00:14:33,927 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:14:33,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:14:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T00:14:34,079 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:34,080 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:14:34,080 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 1ee97d494479a45762cab6f40ed6001d, disabling compactions & flushes 2024-11-13T00:14:34,080 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:34,080 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:34,080 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. after waiting 0 ms 2024-11-13T00:14:34,080 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:34,080 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 1ee97d494479a45762cab6f40ed6001d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T00:14:34,080 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=A 2024-11-13T00:14:34,081 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:34,081 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=B 2024-11-13T00:14:34,081 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:34,081 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1ee97d494479a45762cab6f40ed6001d, store=C 2024-11-13T00:14:34,081 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:34,087 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bbb88a0fea8f4caea93b89564902dd4b_1ee97d494479a45762cab6f40ed6001d is 50, key is test_row_0/A:col10/1731456866766/Put/seqid=0 2024-11-13T00:14:34,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742104_1280 (size=9914) 2024-11-13T00:14:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T00:14:34,491 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:34,495 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bbb88a0fea8f4caea93b89564902dd4b_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bbb88a0fea8f4caea93b89564902dd4b_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:34,496 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/22e4d9485422492bafd3208c9b891d81, store: [table=TestAcidGuarantees family=A region=1ee97d494479a45762cab6f40ed6001d] 2024-11-13T00:14:34,497 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/22e4d9485422492bafd3208c9b891d81 is 175, key is test_row_0/A:col10/1731456866766/Put/seqid=0 2024-11-13T00:14:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742105_1281 (size=22561) 2024-11-13T00:14:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T00:14:34,900 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=572, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/22e4d9485422492bafd3208c9b891d81 2024-11-13T00:14:34,907 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5fdc2a131aa44c6a9cbf99ddd471bc4a is 50, key is test_row_0/B:col10/1731456866766/Put/seqid=0 2024-11-13T00:14:34,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742106_1282 (size=9857) 2024-11-13T00:14:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T00:14:35,311 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5fdc2a131aa44c6a9cbf99ddd471bc4a 2024-11-13T00:14:35,317 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/fd5462c7554c4e6798978dd1dd763050 is 50, key is test_row_0/C:col10/1731456866766/Put/seqid=0 2024-11-13T00:14:35,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742107_1283 (size=9857) 2024-11-13T00:14:35,721 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/fd5462c7554c4e6798978dd1dd763050 2024-11-13T00:14:35,725 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/A/22e4d9485422492bafd3208c9b891d81 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/22e4d9485422492bafd3208c9b891d81 2024-11-13T00:14:35,729 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/22e4d9485422492bafd3208c9b891d81, entries=100, sequenceid=572, filesize=22.0 K 2024-11-13T00:14:35,729 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/B/5fdc2a131aa44c6a9cbf99ddd471bc4a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5fdc2a131aa44c6a9cbf99ddd471bc4a 2024-11-13T00:14:35,732 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5fdc2a131aa44c6a9cbf99ddd471bc4a, entries=100, sequenceid=572, filesize=9.6 K 2024-11-13T00:14:35,733 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/.tmp/C/fd5462c7554c4e6798978dd1dd763050 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/fd5462c7554c4e6798978dd1dd763050 2024-11-13T00:14:35,736 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/fd5462c7554c4e6798978dd1dd763050, entries=100, sequenceid=572, filesize=9.6 K 2024-11-13T00:14:35,737 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1ee97d494479a45762cab6f40ed6001d in 1657ms, sequenceid=572, compaction requested=true 2024-11-13T00:14:35,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/bccc526ba7814dac9be4777975142b82, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/60e080cf96004d83bf927d411dbcb9a2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/9237d423e6514a05ab98652aa4fdac62, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6e210f0b5cb54e318faffd39de07096d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/dc781b8275ae466dbcb6221ab59d16bf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ec27e49b63cc4437a26656100a2f3026, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/57ddf4103a2a4b238355208519c36865, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/df2d5e66d2a84c75b5dd3a2ae558f9c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/e6061cd3819c4b8ea52d76dde3f0e8d6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/849df79e00c24fc4a6b376650171dcc6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a] to archive 2024-11-13T00:14:35,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:14:35,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1011b8f222074026b24a53293cd5659e 2024-11-13T00:14:35,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/96f368467f9f47a8a6731d6bfc1b69b6 2024-11-13T00:14:35,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/bccc526ba7814dac9be4777975142b82 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/bccc526ba7814dac9be4777975142b82 2024-11-13T00:14:35,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/52fecda1679b400ba5bb907ed859241b 2024-11-13T00:14:35,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/04c8639fad3b4910910751c6d2d811e5 2024-11-13T00:14:35,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/60e080cf96004d83bf927d411dbcb9a2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/60e080cf96004d83bf927d411dbcb9a2 2024-11-13T00:14:35,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a299b8e0c608426fb962ac3410d12450 2024-11-13T00:14:35,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6a76ab7ba0994bd182550a88ed375e42 2024-11-13T00:14:35,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/9237d423e6514a05ab98652aa4fdac62 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/9237d423e6514a05ab98652aa4fdac62 2024-11-13T00:14:35,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/d8fd3456fc66459a9c7a78dd410e854e 2024-11-13T00:14:35,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4fb67bdbf5354ddd825a957d738ddbad 2024-11-13T00:14:35,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ae441610d70e4c7f9927da72c4ebdce3 2024-11-13T00:14:35,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6e210f0b5cb54e318faffd39de07096d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/6e210f0b5cb54e318faffd39de07096d 2024-11-13T00:14:35,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/4ea8f519c1b94104ba9462c8abb40c65 2024-11-13T00:14:35,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/dc781b8275ae466dbcb6221ab59d16bf to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/dc781b8275ae466dbcb6221ab59d16bf 2024-11-13T00:14:35,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/48e888d26543429f8e7dcb09d39d0929 2024-11-13T00:14:35,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fc603e611c647e8908e4d92beb21783 2024-11-13T00:14:35,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/5a178a87e6594eedbcc24928854adb39 2024-11-13T00:14:35,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ec27e49b63cc4437a26656100a2f3026 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/ec27e49b63cc4437a26656100a2f3026 2024-11-13T00:14:35,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/699eeba2448d45938c709fea57b4ef37 2024-11-13T00:14:35,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a85b994c51b5472283a2a7bfe94b95f5 2024-11-13T00:14:35,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/57ddf4103a2a4b238355208519c36865 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/57ddf4103a2a4b238355208519c36865 2024-11-13T00:14:35,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8efb9217b0f942df8b44c8d2783b0245 2024-11-13T00:14:35,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/76830db3d7ab4c4982c02ef069e917e2 2024-11-13T00:14:35,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/df2d5e66d2a84c75b5dd3a2ae558f9c6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/df2d5e66d2a84c75b5dd3a2ae558f9c6 2024-11-13T00:14:35,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/1f35c3ae6a554bcfae665451ed904196 2024-11-13T00:14:35,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3bf082f39f58466989eb97295a7c4586 2024-11-13T00:14:35,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/e6061cd3819c4b8ea52d76dde3f0e8d6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/e6061cd3819c4b8ea52d76dde3f0e8d6 2024-11-13T00:14:35,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/34b294c4941f4e4c935549f119f573c7 2024-11-13T00:14:35,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/3d8ec131019f4c6cb882ca2aa6655239 2024-11-13T00:14:35,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/849df79e00c24fc4a6b376650171dcc6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/849df79e00c24fc4a6b376650171dcc6 2024-11-13T00:14:35,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/a39335c2046f48dab40515fc66e8cc40 2024-11-13T00:14:35,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/575643a335064e45ac5a251e080b61c6 2024-11-13T00:14:35,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/2387794d4b654291923891cbbd62884a 2024-11-13T00:14:35,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a10fd20937f84303b0565604934796fe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5a2eca25d9014852b3055b3211633014, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/c6612dbfcdc944c281f24dc66d900aa1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bc762089319648a68f38a657238d6f51, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1cfbcd6307d4520bbed902583e7626a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8874a19ea2ad41cba61b10cb0d506b63, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f813bebb3efc49619964f9a4ebcec822, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/008e306178284858b5c5915a6ffbca21, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9fbf385de1344666b0a423cd7a641441, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/b32e65b7c6a742788f322b992bb503b4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/37b83f56a30244959e2403af2e0ccdac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bbeb3d0a3fdb45ed9461b51d4de5e865, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8c0b2e49ca6249d2b8e59910fdc794a0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1f8740fa55c4f7aad2930dac51962d8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8ad810db0d8c46bb954c2afed063f7a4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d82fead44d7b4931ab52e956878623b0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3be6ee4e830c4e64af6d5e24a96e7ce1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/33dbe666d2cd4a67bcf3bae376d7c39a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/49bb4490ef2347558a95b334828fb54a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/473c92f09ac44630b975937be628f435, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3fc2ed4ead6541778b1aa5f7d2c45f4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d2f93e017d3d4231907961b92b7792c1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a0d5a94b2d554f0dad0e870d6d96706d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fa2c39ab92c64228910afe3d11a35d68, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3368dec63f9d4073a99b8950f4416674, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ddaf7c14510e4554bf4dfc561b577e09, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3e394f1b06004658bc77dcd2d80aa692, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/cca76ef348834aba913edb2a68672bb8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/840e17aeb20b42958d1afe2917f99be8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5f5a39e9e2cb485488d29373cf2aaa46, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ca33b736fc524416954b5b3e9d24fa34, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f8cc712f70f24c649df01d59c2faf873, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06] to archive 2024-11-13T00:14:35,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:14:35,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a10fd20937f84303b0565604934796fe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a10fd20937f84303b0565604934796fe 2024-11-13T00:14:35,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5a2eca25d9014852b3055b3211633014 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5a2eca25d9014852b3055b3211633014 2024-11-13T00:14:35,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/c6612dbfcdc944c281f24dc66d900aa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/c6612dbfcdc944c281f24dc66d900aa1 2024-11-13T00:14:35,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bc762089319648a68f38a657238d6f51 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bc762089319648a68f38a657238d6f51 2024-11-13T00:14:35,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1cfbcd6307d4520bbed902583e7626a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1cfbcd6307d4520bbed902583e7626a 2024-11-13T00:14:35,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8874a19ea2ad41cba61b10cb0d506b63 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8874a19ea2ad41cba61b10cb0d506b63 2024-11-13T00:14:35,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f813bebb3efc49619964f9a4ebcec822 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f813bebb3efc49619964f9a4ebcec822 2024-11-13T00:14:35,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/008e306178284858b5c5915a6ffbca21 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/008e306178284858b5c5915a6ffbca21 2024-11-13T00:14:35,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9fbf385de1344666b0a423cd7a641441 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9fbf385de1344666b0a423cd7a641441 2024-11-13T00:14:35,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/b32e65b7c6a742788f322b992bb503b4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/b32e65b7c6a742788f322b992bb503b4 2024-11-13T00:14:35,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/37b83f56a30244959e2403af2e0ccdac to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/37b83f56a30244959e2403af2e0ccdac 2024-11-13T00:14:35,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bbeb3d0a3fdb45ed9461b51d4de5e865 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/bbeb3d0a3fdb45ed9461b51d4de5e865 2024-11-13T00:14:35,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8c0b2e49ca6249d2b8e59910fdc794a0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8c0b2e49ca6249d2b8e59910fdc794a0 2024-11-13T00:14:35,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1f8740fa55c4f7aad2930dac51962d8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a1f8740fa55c4f7aad2930dac51962d8 2024-11-13T00:14:35,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8ad810db0d8c46bb954c2afed063f7a4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/8ad810db0d8c46bb954c2afed063f7a4 2024-11-13T00:14:35,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d82fead44d7b4931ab52e956878623b0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d82fead44d7b4931ab52e956878623b0 2024-11-13T00:14:35,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3be6ee4e830c4e64af6d5e24a96e7ce1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3be6ee4e830c4e64af6d5e24a96e7ce1 2024-11-13T00:14:35,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/33dbe666d2cd4a67bcf3bae376d7c39a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/33dbe666d2cd4a67bcf3bae376d7c39a 2024-11-13T00:14:35,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/49bb4490ef2347558a95b334828fb54a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/49bb4490ef2347558a95b334828fb54a 2024-11-13T00:14:35,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/473c92f09ac44630b975937be628f435 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/473c92f09ac44630b975937be628f435 2024-11-13T00:14:35,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3fc2ed4ead6541778b1aa5f7d2c45f4c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3fc2ed4ead6541778b1aa5f7d2c45f4c 2024-11-13T00:14:35,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d2f93e017d3d4231907961b92b7792c1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/d2f93e017d3d4231907961b92b7792c1 2024-11-13T00:14:35,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a0d5a94b2d554f0dad0e870d6d96706d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/a0d5a94b2d554f0dad0e870d6d96706d 2024-11-13T00:14:35,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fa2c39ab92c64228910afe3d11a35d68 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fa2c39ab92c64228910afe3d11a35d68 2024-11-13T00:14:35,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3368dec63f9d4073a99b8950f4416674 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3368dec63f9d4073a99b8950f4416674 2024-11-13T00:14:35,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/4c9ebdb6107f4dbc8c50bfbd3dd2825d 2024-11-13T00:14:35,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ddaf7c14510e4554bf4dfc561b577e09 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ddaf7c14510e4554bf4dfc561b577e09 2024-11-13T00:14:35,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3e394f1b06004658bc77dcd2d80aa692 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/3e394f1b06004658bc77dcd2d80aa692 2024-11-13T00:14:35,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/cca76ef348834aba913edb2a68672bb8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/cca76ef348834aba913edb2a68672bb8 2024-11-13T00:14:35,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/840e17aeb20b42958d1afe2917f99be8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/840e17aeb20b42958d1afe2917f99be8 2024-11-13T00:14:35,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5f5a39e9e2cb485488d29373cf2aaa46 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5f5a39e9e2cb485488d29373cf2aaa46 2024-11-13T00:14:35,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ca33b736fc524416954b5b3e9d24fa34 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/ca33b736fc524416954b5b3e9d24fa34 2024-11-13T00:14:35,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f8cc712f70f24c649df01d59c2faf873 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/f8cc712f70f24c649df01d59c2faf873 2024-11-13T00:14:35,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/9b8bc29bb0cb4d6cbdc9d729ab25cf06 2024-11-13T00:14:35,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d58213203244a7dba40af964674ae97, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cc6007d2909046ecba721dff65a0bb67, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1853a7ada461414380f77e7c95bddbf8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/91ea12042c324ebca6a0dd2a963d81aa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df9889c4e934474f83806485ffa3f67f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/53c4e399f8a4409da79a3397452931fe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/690e2b8805f14e6db303a64ecd90f1ad, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ee5cd254b728483e87809939abc66ea2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df75422792b54540ac93f14edd61a58b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a87c3f628bc843e8b48f3d7b834c5f5b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/9029d02c8d5542e4b80bf34196100b6a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/44ee50b01536456c9daea41116743ded, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bff4ce06eff34c4a9000a870cd5be11f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a5b86e953390443889e68af8728a52a3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/d7ea609e0e42479c84cba18b15df3b51, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/aa87d54bcabc4456b54e2598eb879385, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/48bd1222ab6747d8b82cb6b858a65ddd, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bc5ba7f7129b4c61a09d17086792ead0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/33887b4faf384e348d1218c8ad3a3bab, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/b807eb990f8f454ba7def0c69885586a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d8a457cdeb94f9e8a4dedc9dc65b99d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7b358a50b12e4d609bb39b915513634d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/965f9ddf524a438fb4bdbbc8c51c08da, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3799a4ba9b084743a7b08b057632afc0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/738753b932d54e6694be4d701d571aec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3e8be7b2f1dd43468b1c94b07fda5f8e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7a3bbafbc0904431b43876d94052b44a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1cf5ff7f18294b6db912ea9de7f47e27, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ed3f536635634611b2e778e18392aae8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/5702066677e84910ba9a68c87442e776, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f849b96dd37b4c3bb05b3ac5f0e7cf99, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/742beccf67934ac78f61121d51f89d93, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f9b1d5e82a6b4d2885213e7c4e793d2c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cd9afdd437164402a13f533c4f2d5660] to archive 2024-11-13T00:14:35,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:14:35,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d58213203244a7dba40af964674ae97 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d58213203244a7dba40af964674ae97 2024-11-13T00:14:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cc6007d2909046ecba721dff65a0bb67 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cc6007d2909046ecba721dff65a0bb67 2024-11-13T00:14:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1853a7ada461414380f77e7c95bddbf8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1853a7ada461414380f77e7c95bddbf8 2024-11-13T00:14:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/91ea12042c324ebca6a0dd2a963d81aa to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/91ea12042c324ebca6a0dd2a963d81aa 2024-11-13T00:14:35,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df9889c4e934474f83806485ffa3f67f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df9889c4e934474f83806485ffa3f67f 2024-11-13T00:14:35,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/53c4e399f8a4409da79a3397452931fe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/53c4e399f8a4409da79a3397452931fe 2024-11-13T00:14:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/690e2b8805f14e6db303a64ecd90f1ad to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/690e2b8805f14e6db303a64ecd90f1ad 2024-11-13T00:14:35,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ee5cd254b728483e87809939abc66ea2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ee5cd254b728483e87809939abc66ea2 2024-11-13T00:14:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df75422792b54540ac93f14edd61a58b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/df75422792b54540ac93f14edd61a58b 2024-11-13T00:14:35,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a87c3f628bc843e8b48f3d7b834c5f5b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a87c3f628bc843e8b48f3d7b834c5f5b 2024-11-13T00:14:35,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/9029d02c8d5542e4b80bf34196100b6a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/9029d02c8d5542e4b80bf34196100b6a 2024-11-13T00:14:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/44ee50b01536456c9daea41116743ded to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/44ee50b01536456c9daea41116743ded 2024-11-13T00:14:35,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bff4ce06eff34c4a9000a870cd5be11f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bff4ce06eff34c4a9000a870cd5be11f 2024-11-13T00:14:35,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a5b86e953390443889e68af8728a52a3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/a5b86e953390443889e68af8728a52a3 2024-11-13T00:14:35,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/d7ea609e0e42479c84cba18b15df3b51 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/d7ea609e0e42479c84cba18b15df3b51 2024-11-13T00:14:35,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/aa87d54bcabc4456b54e2598eb879385 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/aa87d54bcabc4456b54e2598eb879385 2024-11-13T00:14:35,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/48bd1222ab6747d8b82cb6b858a65ddd to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/48bd1222ab6747d8b82cb6b858a65ddd 2024-11-13T00:14:35,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bc5ba7f7129b4c61a09d17086792ead0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/bc5ba7f7129b4c61a09d17086792ead0 2024-11-13T00:14:35,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/33887b4faf384e348d1218c8ad3a3bab to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/33887b4faf384e348d1218c8ad3a3bab 2024-11-13T00:14:35,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/b807eb990f8f454ba7def0c69885586a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/b807eb990f8f454ba7def0c69885586a 2024-11-13T00:14:35,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d8a457cdeb94f9e8a4dedc9dc65b99d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/4d8a457cdeb94f9e8a4dedc9dc65b99d 2024-11-13T00:14:35,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7b358a50b12e4d609bb39b915513634d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7b358a50b12e4d609bb39b915513634d 2024-11-13T00:14:35,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/965f9ddf524a438fb4bdbbc8c51c08da to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/965f9ddf524a438fb4bdbbc8c51c08da 2024-11-13T00:14:35,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3799a4ba9b084743a7b08b057632afc0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3799a4ba9b084743a7b08b057632afc0 2024-11-13T00:14:35,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/738753b932d54e6694be4d701d571aec to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/738753b932d54e6694be4d701d571aec 2024-11-13T00:14:35,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3e8be7b2f1dd43468b1c94b07fda5f8e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/3e8be7b2f1dd43468b1c94b07fda5f8e 2024-11-13T00:14:35,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7a3bbafbc0904431b43876d94052b44a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/7a3bbafbc0904431b43876d94052b44a 2024-11-13T00:14:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1cf5ff7f18294b6db912ea9de7f47e27 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/1cf5ff7f18294b6db912ea9de7f47e27 2024-11-13T00:14:35,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ed3f536635634611b2e778e18392aae8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/ed3f536635634611b2e778e18392aae8 2024-11-13T00:14:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/5702066677e84910ba9a68c87442e776 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/5702066677e84910ba9a68c87442e776 2024-11-13T00:14:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f849b96dd37b4c3bb05b3ac5f0e7cf99 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f849b96dd37b4c3bb05b3ac5f0e7cf99 2024-11-13T00:14:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/742beccf67934ac78f61121d51f89d93 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/742beccf67934ac78f61121d51f89d93 2024-11-13T00:14:35,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f9b1d5e82a6b4d2885213e7c4e793d2c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/f9b1d5e82a6b4d2885213e7c4e793d2c 2024-11-13T00:14:35,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cd9afdd437164402a13f533c4f2d5660 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/cd9afdd437164402a13f533c4f2d5660 2024-11-13T00:14:35,846 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/recovered.edits/575.seqid, newMaxSeqId=575, maxSeqId=4 2024-11-13T00:14:35,846 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. 2024-11-13T00:14:35,846 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 1ee97d494479a45762cab6f40ed6001d: Waiting for close lock at 1731456874080Running coprocessor pre-close hooks at 1731456874080Disabling compacts and flushes for region at 1731456874080Disabling writes for close at 1731456874080Obtaining lock to block concurrent updates at 1731456874080Preparing flush snapshotting stores in 1ee97d494479a45762cab6f40ed6001d at 1731456874080Finished memstore snapshotting TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731456874081 (+1 ms)Flushing stores of TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d. at 1731456874081Flushing 1ee97d494479a45762cab6f40ed6001d/A: creating writer at 1731456874081Flushing mob file 1ee97d494479a45762cab6f40ed6001d/A: appending metadata at 1731456874086 (+5 ms)Flushing mob file 1ee97d494479a45762cab6f40ed6001d/A: closing flushed file at 1731456874086Flushing 1ee97d494479a45762cab6f40ed6001d/A: appending metadata at 1731456874496 (+410 ms)Flushing 1ee97d494479a45762cab6f40ed6001d/A: closing flushed file at 1731456874496Flushing 1ee97d494479a45762cab6f40ed6001d/B: creating writer at 1731456874904 (+408 ms)Flushing 1ee97d494479a45762cab6f40ed6001d/B: appending metadata at 1731456874906 (+2 ms)Flushing 1ee97d494479a45762cab6f40ed6001d/B: closing flushed file at 1731456874907 (+1 ms)Flushing 1ee97d494479a45762cab6f40ed6001d/C: creating writer at 1731456875314 (+407 ms)Flushing 1ee97d494479a45762cab6f40ed6001d/C: appending metadata at 1731456875317 (+3 ms)Flushing 1ee97d494479a45762cab6f40ed6001d/C: closing flushed file at 1731456875317Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b9d1f07: reopening flushed file at 1731456875725 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689e4ab9: reopening flushed file at 1731456875729 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c7c6796: reopening flushed file at 1731456875733 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1ee97d494479a45762cab6f40ed6001d in 1657ms, sequenceid=572, compaction requested=true at 1731456875737 (+4 ms)Writing region close event to WAL at 1731456875843 (+106 ms)Running coprocessor post-close hooks at 1731456875846 (+3 ms)Closed at 1731456875846 2024-11-13T00:14:35,848 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:35,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1ee97d494479a45762cab6f40ed6001d, regionState=CLOSED 2024-11-13T00:14:35,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:14:35,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T00:14:35,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 1ee97d494479a45762cab6f40ed6001d, server=cb28580903fb,45273,1731456804594 in 1.9240 sec 2024-11-13T00:14:35,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-13T00:14:35,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1ee97d494479a45762cab6f40ed6001d, UNASSIGN in 1.9270 sec 2024-11-13T00:14:35,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T00:14:35,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9300 sec 2024-11-13T00:14:35,856 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456875856"}]},"ts":"1731456875856"} 2024-11-13T00:14:35,857 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T00:14:35,857 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T00:14:35,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-13T00:14:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T00:14:36,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:36,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T00:14:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,069 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T00:14:36,070 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,073 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,075 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/recovered.edits] 2024-11-13T00:14:36,077 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/22e4d9485422492bafd3208c9b891d81 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/22e4d9485422492bafd3208c9b891d81 2024-11-13T00:14:36,078 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/411444d407cf4c0ca3f36a3daa7f1365 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/411444d407cf4c0ca3f36a3daa7f1365 2024-11-13T00:14:36,079 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fb76c9072de4ea981afea49f4e024b5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/A/8fb76c9072de4ea981afea49f4e024b5 2024-11-13T00:14:36,081 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/569a947d5d774bc4a6baa1992b864844 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/569a947d5d774bc4a6baa1992b864844 2024-11-13T00:14:36,082 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5fdc2a131aa44c6a9cbf99ddd471bc4a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/5fdc2a131aa44c6a9cbf99ddd471bc4a 2024-11-13T00:14:36,083 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fd99456f4a334a15a4a9e457d2e56757 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/B/fd99456f4a334a15a4a9e457d2e56757 2024-11-13T00:14:36,085 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/412e9b608f0941eeb24e4a4759c43bda to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/412e9b608f0941eeb24e4a4759c43bda 2024-11-13T00:14:36,086 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/998195d6a3714c87aa8f7ed393d2632d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/998195d6a3714c87aa8f7ed393d2632d 2024-11-13T00:14:36,087 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/fd5462c7554c4e6798978dd1dd763050 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/C/fd5462c7554c4e6798978dd1dd763050 2024-11-13T00:14:36,090 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/recovered.edits/575.seqid to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d/recovered.edits/575.seqid 2024-11-13T00:14:36,090 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,090 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T00:14:36,091 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T00:14:36,091 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T00:14:36,095 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130888ce3ad9ae494eb30b5e5dd3ea5d2f_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130888ce3ad9ae494eb30b5e5dd3ea5d2f_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,096 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308f70177cd984b02a4f0b19859181ba0_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308f70177cd984b02a4f0b19859181ba0_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,097 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131ad679e885a3431191013bdf50176a1d_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131ad679e885a3431191013bdf50176a1d_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,098 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323e49a202cad4f73a05065dd878c2ab1_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323e49a202cad4f73a05065dd878c2ab1_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,099 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132646b9e9662340779003d971e0e9d2a6_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132646b9e9662340779003d971e0e9d2a6_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,100 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132a1fc0d7d88f4a58aa53e5fa56738de2_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132a1fc0d7d88f4a58aa53e5fa56738de2_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,101 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f5d373ab3164de693f690ca5b087794_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f5d373ab3164de693f690ca5b087794_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,102 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133084cf257ef44b06bad00e2cf46df497_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133084cf257ef44b06bad00e2cf46df497_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,104 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134fb76c809a2f4d22b3fee835b63d74b0_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134fb76c809a2f4d22b3fee835b63d74b0_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,106 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113571fabed26404850813992f0d58f6705_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113571fabed26404850813992f0d58f6705_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,107 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111365b6dbb838034fd3ae2c74142eacfb0a_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111365b6dbb838034fd3ae2c74142eacfb0a_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,108 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136977205d40cb4a6ba919fd65df561d0c_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136977205d40cb4a6ba919fd65df561d0c_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,109 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136a97fbd5ffdb43a393eb1fff57938c83_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136a97fbd5ffdb43a393eb1fff57938c83_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,110 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ef354ba5ac9447396aa8e0e9cb6ba69_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ef354ba5ac9447396aa8e0e9cb6ba69_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,112 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113704a87a4300f402885d64204a5327c91_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113704a87a4300f402885d64204a5327c91_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,113 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113721b3b5050094c10b971f3b5b2c559cc_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113721b3b5050094c10b971f3b5b2c559cc_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,114 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137de709a1af2249cfa024aa966a3a99fe_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137de709a1af2249cfa024aa966a3a99fe_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,115 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113817c397cdfbf4e2abcc4311ce49cad4b_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113817c397cdfbf4e2abcc4311ce49cad4b_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,116 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139cc70d38de124f2a9b233111ee1b27bd_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139cc70d38de124f2a9b233111ee1b27bd_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,117 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a286dad0640b425c98ea1dc63c0343db_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a286dad0640b425c98ea1dc63c0343db_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,122 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bbb88a0fea8f4caea93b89564902dd4b_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bbb88a0fea8f4caea93b89564902dd4b_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,123 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c0892c5304ae4f8fb1cca9ebf57fe18f_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c0892c5304ae4f8fb1cca9ebf57fe18f_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,125 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5e172ab3bf34d33ae72ce9f131d684a_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5e172ab3bf34d33ae72ce9f131d684a_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,126 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d8777b5675c04cc2be82e76c15aea8b8_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d8777b5675c04cc2be82e76c15aea8b8_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,129 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e920eb0df9cb4138b8bf84adace62274_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e920eb0df9cb4138b8bf84adace62274_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fd68544d3f1d4c7280ed7a50c0391548_1ee97d494479a45762cab6f40ed6001d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fd68544d3f1d4c7280ed7a50c0391548_1ee97d494479a45762cab6f40ed6001d 2024-11-13T00:14:36,131 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T00:14:36,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,136 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T00:14:36,138 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T00:14:36,139 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,139 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T00:14:36,140 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731456876139"}]},"ts":"9223372036854775807"} 2024-11-13T00:14:36,142 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T00:14:36,142 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1ee97d494479a45762cab6f40ed6001d, NAME => 'TestAcidGuarantees,,1731456842933.1ee97d494479a45762cab6f40ed6001d.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T00:14:36,142 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T00:14:36,142 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731456876142"}]},"ts":"9223372036854775807"} 2024-11-13T00:14:36,144 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T00:14:36,144 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-13T00:14:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T00:14:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T00:14:36,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:36,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1970940896_22 at /127.0.0.1:48542 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1563 (was 2076) 2024-11-13T00:14:36,201 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=1561 2024-11-13T00:14:36,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:14:36,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:14:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T00:14:36,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:14:36,204 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:36,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-13T00:14:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T00:14:36,205 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:14:36,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742108_1284 (size=1061) 2024-11-13T00:14:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T00:14:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T00:14:36,613 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:14:36,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742109_1285 (size=53) 2024-11-13T00:14:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T00:14:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:14:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dc9547c78a88f5a8e277773a4b75b850, disabling compactions & flushes 2024-11-13T00:14:37,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. after waiting 0 ms 2024-11-13T00:14:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc9547c78a88f5a8e277773a4b75b850: Waiting for close lock at 1731456877019Disabling compacts and flushes for region at 1731456877019Disabling writes for close at 1731456877019Writing region close event to WAL at 1731456877019Closed at 1731456877019 2024-11-13T00:14:37,020 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:14:37,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731456877020"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731456877020"}]},"ts":"1731456877020"} 2024-11-13T00:14:37,022 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:14:37,023 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:14:37,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456877023"}]},"ts":"1731456877023"} 2024-11-13T00:14:37,024 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T00:14:37,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, ASSIGN}] 2024-11-13T00:14:37,026 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, ASSIGN 2024-11-13T00:14:37,026 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, ASSIGN; state=OFFLINE, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=false 2024-11-13T00:14:37,177 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=dc9547c78a88f5a8e277773a4b75b850, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:37,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, ASSIGN because future has completed 2024-11-13T00:14:37,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:14:37,333 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,333 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:14:37,334 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,334 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:14:37,334 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,334 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,335 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,336 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:37,336 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc9547c78a88f5a8e277773a4b75b850 columnFamilyName A 2024-11-13T00:14:37,336 DEBUG [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:37,337 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.HStore(327): Store=dc9547c78a88f5a8e277773a4b75b850/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:37,337 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,338 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:37,338 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc9547c78a88f5a8e277773a4b75b850 columnFamilyName B 2024-11-13T00:14:37,338 DEBUG [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:37,338 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.HStore(327): Store=dc9547c78a88f5a8e277773a4b75b850/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:37,339 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,339 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:14:37,339 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc9547c78a88f5a8e277773a4b75b850 columnFamilyName C 2024-11-13T00:14:37,340 DEBUG [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:14:37,340 INFO [StoreOpener-dc9547c78a88f5a8e277773a4b75b850-1 {}] regionserver.HStore(327): Store=dc9547c78a88f5a8e277773a4b75b850/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:14:37,340 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,340 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,341 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,341 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,342 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,342 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,342 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:14:37,343 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,344 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:14:37,345 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened dc9547c78a88f5a8e277773a4b75b850; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69815066, jitterRate=0.040325552225112915}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:14:37,345 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:37,346 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for dc9547c78a88f5a8e277773a4b75b850: Running coprocessor pre-open hook at 1731456877334Writing region info on filesystem at 1731456877334Initializing all the Stores at 1731456877335 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456877335Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456877335Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456877335Cleaning up temporary data from old regions at 1731456877342 (+7 ms)Running coprocessor post-open hooks at 1731456877345 (+3 ms)Region opened successfully at 1731456877346 (+1 ms) 2024-11-13T00:14:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T00:14:37,346 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., pid=67, masterSystemTime=1731456877330 2024-11-13T00:14:37,348 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,348 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:37,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=dc9547c78a88f5a8e277773a4b75b850, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:14:37,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:14:37,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T00:14:37,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 in 172 msec 2024-11-13T00:14:37,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-13T00:14:37,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, ASSIGN in 327 msec 2024-11-13T00:14:37,354 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:14:37,355 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456877354"}]},"ts":"1731456877354"} 2024-11-13T00:14:37,356 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T00:14:37,356 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:14:37,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-13T00:14:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T00:14:38,356 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:38,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100b0271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f94bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,363 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:14:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:14:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0f0acc 2024-11-13T00:14:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:14:38,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:14:38,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea6a7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e9d19c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dacfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,371 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fe1dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e12caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d1002, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b64044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,376 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b7c10d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,378 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6708cd10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,379 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d511b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77fc8128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29508c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,383 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a656b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,384 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,386 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22501971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,388 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785fa60c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,389 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5691a901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:14:38,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:14:38,392 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:14:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:14:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:14:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3b8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:14:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:14:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:38,393 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:14:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@484b36e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:14:38,394 DEBUG [Thread-1299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,394 DEBUG [Thread-1312 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,394 DEBUG [Thread-1314 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,395 DEBUG [Thread-1310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,395 DEBUG [Thread-1318 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,396 DEBUG [Thread-1316 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,397 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:38,398 DEBUG [Thread-1303 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,398 DEBUG [Thread-1301 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,399 DEBUG [Thread-1305 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T00:14:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,399 DEBUG [Thread-1301 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,399 DEBUG [Thread-1301 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,399 DEBUG [Thread-1305 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,400 DEBUG [Thread-1307 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:14:38,400 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,401 DEBUG [Thread-1305 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,401 DEBUG [Thread-1307 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:14:38,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:38,401 DEBUG [Thread-1307 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:14:38,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T00:14:38,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:14:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:14:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:38,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:14:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:38,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:38,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a1fd73b1dec04e9ca106ba550a48b3a4 is 50, key is test_row_0/A:col10/1731456878406/Put/seqid=0 2024-11-13T00:14:38,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742110_1286 (size=14341) 2024-11-13T00:14:38,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a1fd73b1dec04e9ca106ba550a48b3a4 2024-11-13T00:14:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456938446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456938447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456938448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456938449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456938451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/316dc7bd98b34abda6ba9accce52e823 is 50, key is test_row_0/B:col10/1731456878406/Put/seqid=0 2024-11-13T00:14:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T00:14:38,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742111_1287 (size=12001) 2024-11-13T00:14:38,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/316dc7bd98b34abda6ba9accce52e823 2024-11-13T00:14:38,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/0e351485c6cf497fb8da825570ebadd6 is 50, key is test_row_0/C:col10/1731456878406/Put/seqid=0 2024-11-13T00:14:38,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T00:14:38,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:38,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:38,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:38,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742112_1288 (size=12001) 2024-11-13T00:14:38,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456938565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456938566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456938566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456938566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456938566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T00:14:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:38,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T00:14:38,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456938776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456938776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456938776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456938776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456938776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:38,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T00:14:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:38,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:38,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/0e351485c6cf497fb8da825570ebadd6 2024-11-13T00:14:38,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a1fd73b1dec04e9ca106ba550a48b3a4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1fd73b1dec04e9ca106ba550a48b3a4 2024-11-13T00:14:38,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1fd73b1dec04e9ca106ba550a48b3a4, entries=200, sequenceid=13, filesize=14.0 K 2024-11-13T00:14:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/316dc7bd98b34abda6ba9accce52e823 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/316dc7bd98b34abda6ba9accce52e823 2024-11-13T00:14:38,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/316dc7bd98b34abda6ba9accce52e823, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T00:14:38,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/0e351485c6cf497fb8da825570ebadd6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/0e351485c6cf497fb8da825570ebadd6 2024-11-13T00:14:38,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/0e351485c6cf497fb8da825570ebadd6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T00:14:38,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dc9547c78a88f5a8e277773a4b75b850 in 564ms, sequenceid=13, compaction requested=false 2024-11-13T00:14:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:39,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:39,017 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:39,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/086bb8168e9146c594ede89f7c4b43ee is 50, key is test_row_0/A:col10/1731456878449/Put/seqid=0 2024-11-13T00:14:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T00:14:39,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742113_1289 (size=12001) 2024-11-13T00:14:39,032 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/086bb8168e9146c594ede89f7c4b43ee 2024-11-13T00:14:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a2b1c968b517469e8ca56747452d7464 is 50, key is test_row_0/B:col10/1731456878449/Put/seqid=0 2024-11-13T00:14:39,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742114_1290 (size=12001) 2024-11-13T00:14:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:39,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456939107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456939107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456939108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456939108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456939108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456939216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456939216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456939216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456939216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456939216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456939426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456939426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456939426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456939426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456939426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,455 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a2b1c968b517469e8ca56747452d7464 2024-11-13T00:14:39,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e1e60dec84d341f5a5b783aa918e634e is 50, key is test_row_0/C:col10/1731456878449/Put/seqid=0 2024-11-13T00:14:39,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742115_1291 (size=12001) 2024-11-13T00:14:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T00:14:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456939736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456939736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456939736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456939737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456939737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:39,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:39,867 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e1e60dec84d341f5a5b783aa918e634e 2024-11-13T00:14:39,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/086bb8168e9146c594ede89f7c4b43ee as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/086bb8168e9146c594ede89f7c4b43ee 2024-11-13T00:14:39,877 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/086bb8168e9146c594ede89f7c4b43ee, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T00:14:39,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a2b1c968b517469e8ca56747452d7464 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a2b1c968b517469e8ca56747452d7464 2024-11-13T00:14:39,882 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a2b1c968b517469e8ca56747452d7464, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T00:14:39,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e1e60dec84d341f5a5b783aa918e634e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e1e60dec84d341f5a5b783aa918e634e 2024-11-13T00:14:39,887 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e1e60dec84d341f5a5b783aa918e634e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T00:14:39,888 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for dc9547c78a88f5a8e277773a4b75b850 in 871ms, sequenceid=33, compaction requested=false 2024-11-13T00:14:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T00:14:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T00:14:39,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T00:14:39,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-11-13T00:14:39,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4960 sec 2024-11-13T00:14:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:40,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:14:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:40,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/7678836679ed4eaf9f487dc5acf4e357 is 50, key is test_row_0/A:col10/1731456880247/Put/seqid=0 2024-11-13T00:14:40,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742116_1292 (size=12001) 2024-11-13T00:14:40,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/7678836679ed4eaf9f487dc5acf4e357 2024-11-13T00:14:40,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456940266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456940267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456940269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456940269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456940271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e5a5f8813be14bf590a03ceaacebbf3e is 50, key is test_row_0/B:col10/1731456880247/Put/seqid=0 2024-11-13T00:14:40,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742117_1293 (size=12001) 2024-11-13T00:14:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456940375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456940386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456940386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456940386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456940386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T00:14:40,546 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:40,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T00:14:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T00:14:40,549 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:40,550 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:40,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:40,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456940586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456940596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456940596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456940596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456940596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T00:14:40,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e5a5f8813be14bf590a03ceaacebbf3e 2024-11-13T00:14:40,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/d9a04bfe3a58436a988f1abda68f07c2 is 50, key is test_row_0/C:col10/1731456880247/Put/seqid=0 2024-11-13T00:14:40,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742118_1294 (size=12001) 2024-11-13T00:14:40,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/d9a04bfe3a58436a988f1abda68f07c2 2024-11-13T00:14:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/7678836679ed4eaf9f487dc5acf4e357 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7678836679ed4eaf9f487dc5acf4e357 2024-11-13T00:14:40,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T00:14:40,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:40,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:40,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7678836679ed4eaf9f487dc5acf4e357, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T00:14:40,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e5a5f8813be14bf590a03ceaacebbf3e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5a5f8813be14bf590a03ceaacebbf3e 2024-11-13T00:14:40,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5a5f8813be14bf590a03ceaacebbf3e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T00:14:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/d9a04bfe3a58436a988f1abda68f07c2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/d9a04bfe3a58436a988f1abda68f07c2 2024-11-13T00:14:40,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/d9a04bfe3a58436a988f1abda68f07c2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T00:14:40,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for dc9547c78a88f5a8e277773a4b75b850 in 469ms, sequenceid=48, compaction requested=true 2024-11-13T00:14:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:40,718 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:40,718 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:40,719 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:40,719 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:40,719 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:40,719 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:40,719 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:40,719 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:40,719 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/316dc7bd98b34abda6ba9accce52e823, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a2b1c968b517469e8ca56747452d7464, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5a5f8813be14bf590a03ceaacebbf3e] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.2 K 2024-11-13T00:14:40,719 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1fd73b1dec04e9ca106ba550a48b3a4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/086bb8168e9146c594ede89f7c4b43ee, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7678836679ed4eaf9f487dc5acf4e357] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=37.4 K 2024-11-13T00:14:40,719 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 316dc7bd98b34abda6ba9accce52e823, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731456878406 2024-11-13T00:14:40,719 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1fd73b1dec04e9ca106ba550a48b3a4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731456878406 2024-11-13T00:14:40,720 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b1c968b517469e8ca56747452d7464, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731456878443 2024-11-13T00:14:40,720 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086bb8168e9146c594ede89f7c4b43ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731456878443 2024-11-13T00:14:40,720 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a5f8813be14bf590a03ceaacebbf3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731456879105 2024-11-13T00:14:40,720 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7678836679ed4eaf9f487dc5acf4e357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731456879105 2024-11-13T00:14:40,730 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:40,730 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:40,731 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ff6d9a87d82e44aa93315883cb570e1a is 50, key is test_row_0/B:col10/1731456880247/Put/seqid=0 2024-11-13T00:14:40,731 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/80bb50bf285d410f976e88a0d783f62f is 50, key is test_row_0/A:col10/1731456880247/Put/seqid=0 2024-11-13T00:14:40,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742120_1296 (size=12104) 2024-11-13T00:14:40,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742119_1295 (size=12104) 2024-11-13T00:14:40,759 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ff6d9a87d82e44aa93315883cb570e1a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ff6d9a87d82e44aa93315883cb570e1a 2024-11-13T00:14:40,763 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into ff6d9a87d82e44aa93315883cb570e1a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:40,763 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:40,763 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=13, startTime=1731456880718; duration=0sec 2024-11-13T00:14:40,763 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:40,763 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:40,763 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:40,764 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:40,764 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:40,764 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:40,764 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/0e351485c6cf497fb8da825570ebadd6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e1e60dec84d341f5a5b783aa918e634e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/d9a04bfe3a58436a988f1abda68f07c2] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.2 K 2024-11-13T00:14:40,765 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e351485c6cf497fb8da825570ebadd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731456878406 2024-11-13T00:14:40,765 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e60dec84d341f5a5b783aa918e634e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731456878443 2024-11-13T00:14:40,766 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a04bfe3a58436a988f1abda68f07c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731456879105 2024-11-13T00:14:40,773 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:40,774 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/51375b48228f4f09a25e5f28d45b2050 is 50, key is test_row_0/C:col10/1731456880247/Put/seqid=0 2024-11-13T00:14:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742121_1297 (size=12104) 2024-11-13T00:14:40,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:40,858 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:40,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/935bce0da3f6498b8750d48ca1cd2af2 is 50, key is test_row_0/A:col10/1731456880266/Put/seqid=0 2024-11-13T00:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T00:14:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742122_1298 (size=12001) 2024-11-13T00:14:40,882 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/935bce0da3f6498b8750d48ca1cd2af2 2024-11-13T00:14:40,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/685d3db5ed134d829dc79e4d4f5160ba is 50, key is test_row_0/B:col10/1731456880266/Put/seqid=0 2024-11-13T00:14:40,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742123_1299 (size=12001) 2024-11-13T00:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:40,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456940915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456940917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456940918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456940919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456940921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:40,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456941026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456941026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456941026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456941035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456941036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,158 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/80bb50bf285d410f976e88a0d783f62f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/80bb50bf285d410f976e88a0d783f62f 2024-11-13T00:14:41,166 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 80bb50bf285d410f976e88a0d783f62f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:41,166 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:41,166 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=13, startTime=1731456880717; duration=0sec 2024-11-13T00:14:41,166 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:41,166 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T00:14:41,197 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/51375b48228f4f09a25e5f28d45b2050 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/51375b48228f4f09a25e5f28d45b2050 2024-11-13T00:14:41,214 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 51375b48228f4f09a25e5f28d45b2050(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:41,214 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:41,214 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=13, startTime=1731456880718; duration=0sec 2024-11-13T00:14:41,214 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:41,214 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:41,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456941236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456941236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456941236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456941246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456941248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,298 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/685d3db5ed134d829dc79e4d4f5160ba 2024-11-13T00:14:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b2a3b74156964a01b61a58ac8d762ee9 is 50, key is test_row_0/C:col10/1731456880266/Put/seqid=0 2024-11-13T00:14:41,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742124_1300 (size=12001) 2024-11-13T00:14:41,372 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b2a3b74156964a01b61a58ac8d762ee9 2024-11-13T00:14:41,374 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:14:41,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/935bce0da3f6498b8750d48ca1cd2af2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/935bce0da3f6498b8750d48ca1cd2af2 2024-11-13T00:14:41,397 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/935bce0da3f6498b8750d48ca1cd2af2, entries=150, sequenceid=66, filesize=11.7 K 2024-11-13T00:14:41,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/685d3db5ed134d829dc79e4d4f5160ba as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/685d3db5ed134d829dc79e4d4f5160ba 2024-11-13T00:14:41,405 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/685d3db5ed134d829dc79e4d4f5160ba, entries=150, sequenceid=66, filesize=11.7 K 2024-11-13T00:14:41,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b2a3b74156964a01b61a58ac8d762ee9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b2a3b74156964a01b61a58ac8d762ee9 2024-11-13T00:14:41,417 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b2a3b74156964a01b61a58ac8d762ee9, entries=150, sequenceid=66, filesize=11.7 K 2024-11-13T00:14:41,418 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for dc9547c78a88f5a8e277773a4b75b850 in 560ms, sequenceid=66, compaction requested=false 2024-11-13T00:14:41,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:41,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:41,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T00:14:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T00:14:41,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T00:14:41,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-11-13T00:14:41,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 880 msec 2024-11-13T00:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:41,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T00:14:41,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:41,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:41,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:41,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:41,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:41,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:41,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/797b01d96fb447b4ad540b2cbe4956fe is 50, key is test_row_0/A:col10/1731456881551/Put/seqid=0 2024-11-13T00:14:41,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742125_1301 (size=12001) 2024-11-13T00:14:41,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/797b01d96fb447b4ad540b2cbe4956fe 2024-11-13T00:14:41,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/b51feb39c6c94f4a90cfccb8a58fed1e is 50, key is test_row_0/B:col10/1731456881551/Put/seqid=0 2024-11-13T00:14:41,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456941626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456941633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456941633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456941633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456941635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742126_1302 (size=12001) 2024-11-13T00:14:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T00:14:41,686 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:41,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T00:14:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:41,690 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:41,690 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:41,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456941746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456941746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456941756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456941756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456941757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:41,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T00:14:41,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:41,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:41,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:41,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:41,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456941956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456941956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456941967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456941967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456941967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:41,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T00:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:41,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:42,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/b51feb39c6c94f4a90cfccb8a58fed1e 2024-11-13T00:14:42,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e74945a9a9b84f5cb718a5d01f4f2b5b is 50, key is test_row_0/C:col10/1731456881551/Put/seqid=0 2024-11-13T00:14:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742127_1303 (size=12001) 2024-11-13T00:14:42,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T00:14:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456942266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456942266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456942276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456942276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456942276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T00:14:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:42,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T00:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:42,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e74945a9a9b84f5cb718a5d01f4f2b5b 2024-11-13T00:14:42,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/797b01d96fb447b4ad540b2cbe4956fe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/797b01d96fb447b4ad540b2cbe4956fe 2024-11-13T00:14:42,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/797b01d96fb447b4ad540b2cbe4956fe, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T00:14:42,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/b51feb39c6c94f4a90cfccb8a58fed1e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b51feb39c6c94f4a90cfccb8a58fed1e 2024-11-13T00:14:42,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b51feb39c6c94f4a90cfccb8a58fed1e, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T00:14:42,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e74945a9a9b84f5cb718a5d01f4f2b5b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e74945a9a9b84f5cb718a5d01f4f2b5b 2024-11-13T00:14:42,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e74945a9a9b84f5cb718a5d01f4f2b5b, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T00:14:42,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for dc9547c78a88f5a8e277773a4b75b850 in 942ms, sequenceid=86, compaction requested=true 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:42,494 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:42,494 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:42,495 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:42,495 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:42,495 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:42,495 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:42,495 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,495 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,495 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/80bb50bf285d410f976e88a0d783f62f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/935bce0da3f6498b8750d48ca1cd2af2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/797b01d96fb447b4ad540b2cbe4956fe] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.3 K 2024-11-13T00:14:42,495 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ff6d9a87d82e44aa93315883cb570e1a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/685d3db5ed134d829dc79e4d4f5160ba, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b51feb39c6c94f4a90cfccb8a58fed1e] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.3 K 2024-11-13T00:14:42,496 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80bb50bf285d410f976e88a0d783f62f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731456879105 2024-11-13T00:14:42,496 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6d9a87d82e44aa93315883cb570e1a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731456879105 2024-11-13T00:14:42,496 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 935bce0da3f6498b8750d48ca1cd2af2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731456880266 2024-11-13T00:14:42,496 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 685d3db5ed134d829dc79e4d4f5160ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731456880266 2024-11-13T00:14:42,497 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 797b01d96fb447b4ad540b2cbe4956fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731456880913 2024-11-13T00:14:42,497 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b51feb39c6c94f4a90cfccb8a58fed1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731456880913 2024-11-13T00:14:42,505 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:42,506 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/712e5a09341a4da78d9e2ab546bfa5a6 is 50, key is test_row_0/A:col10/1731456881551/Put/seqid=0 2024-11-13T00:14:42,506 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:42,506 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/8834835a36ce45969c0e96451cb9e491 is 50, key is test_row_0/B:col10/1731456881551/Put/seqid=0 2024-11-13T00:14:42,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742128_1304 (size=12207) 2024-11-13T00:14:42,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742129_1305 (size=12207) 2024-11-13T00:14:42,517 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/712e5a09341a4da78d9e2ab546bfa5a6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/712e5a09341a4da78d9e2ab546bfa5a6 2024-11-13T00:14:42,520 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/8834835a36ce45969c0e96451cb9e491 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/8834835a36ce45969c0e96451cb9e491 2024-11-13T00:14:42,522 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 712e5a09341a4da78d9e2ab546bfa5a6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:42,522 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:42,522 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=13, startTime=1731456882494; duration=0sec 2024-11-13T00:14:42,522 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:42,522 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:42,522 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:42,524 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:42,524 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:42,524 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,525 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/51375b48228f4f09a25e5f28d45b2050, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b2a3b74156964a01b61a58ac8d762ee9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e74945a9a9b84f5cb718a5d01f4f2b5b] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.3 K 2024-11-13T00:14:42,525 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51375b48228f4f09a25e5f28d45b2050, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731456879105 2024-11-13T00:14:42,525 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2a3b74156964a01b61a58ac8d762ee9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731456880266 2024-11-13T00:14:42,526 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74945a9a9b84f5cb718a5d01f4f2b5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731456880913 2024-11-13T00:14:42,527 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into 8834835a36ce45969c0e96451cb9e491(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:42,527 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:42,527 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=13, startTime=1731456882494; duration=0sec 2024-11-13T00:14:42,527 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:42,527 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:42,540 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:42,541 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/64ecb204bbef45ee8a1b1e12dddaf874 is 50, key is test_row_0/C:col10/1731456881551/Put/seqid=0 2024-11-13T00:14:42,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742130_1306 (size=12207) 2024-11-13T00:14:42,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T00:14:42,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:42,612 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:14:42,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:42,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:42,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:42,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/ced4f2979e9345c98511bbcb2a1a03d1 is 50, key is test_row_0/A:col10/1731456881631/Put/seqid=0 2024-11-13T00:14:42,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742131_1307 (size=12001) 2024-11-13T00:14:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456942795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456942797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456942799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456942800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456942801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456942905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456942905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456942906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456942916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456942916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:42,954 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/64ecb204bbef45ee8a1b1e12dddaf874 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/64ecb204bbef45ee8a1b1e12dddaf874 2024-11-13T00:14:42,959 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 64ecb204bbef45ee8a1b1e12dddaf874(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:42,959 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:42,959 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=13, startTime=1731456882494; duration=0sec 2024-11-13T00:14:42,959 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:42,959 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:43,022 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/ced4f2979e9345c98511bbcb2a1a03d1 2024-11-13T00:14:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/435d1bde48ef41b0a25ba24b7494081c is 50, key is test_row_0/B:col10/1731456881631/Put/seqid=0 2024-11-13T00:14:43,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742132_1308 (size=12001) 2024-11-13T00:14:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456943116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456943116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456943116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456943126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456943126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456943426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456943426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456943427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,433 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/435d1bde48ef41b0a25ba24b7494081c 2024-11-13T00:14:43,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456943436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456943436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/82a132bc78074c549632847852eea940 is 50, key is test_row_0/C:col10/1731456881631/Put/seqid=0 2024-11-13T00:14:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742133_1309 (size=12001) 2024-11-13T00:14:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:43,846 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/82a132bc78074c549632847852eea940 2024-11-13T00:14:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/ced4f2979e9345c98511bbcb2a1a03d1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ced4f2979e9345c98511bbcb2a1a03d1 2024-11-13T00:14:43,855 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ced4f2979e9345c98511bbcb2a1a03d1, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T00:14:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/435d1bde48ef41b0a25ba24b7494081c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/435d1bde48ef41b0a25ba24b7494081c 2024-11-13T00:14:43,864 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/435d1bde48ef41b0a25ba24b7494081c, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T00:14:43,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/82a132bc78074c549632847852eea940 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/82a132bc78074c549632847852eea940 2024-11-13T00:14:43,869 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/82a132bc78074c549632847852eea940, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T00:14:43,870 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for dc9547c78a88f5a8e277773a4b75b850 in 1258ms, sequenceid=102, compaction requested=false 2024-11-13T00:14:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T00:14:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T00:14:43,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T00:14:43,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-13T00:14:43,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.1880 sec 2024-11-13T00:14:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:43,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:14:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:43,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9 is 50, key is test_row_0/A:col10/1731456883938/Put/seqid=0 2024-11-13T00:14:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742134_1310 (size=14341) 2024-11-13T00:14:43,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9 2024-11-13T00:14:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456943951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456943953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456943953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456943955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456943956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/0100d60289044138a0b90e262296bf8f is 50, key is test_row_0/B:col10/1731456883938/Put/seqid=0 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:43,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742135_1311 (size=12001) 2024-11-13T00:14:43,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/0100d60289044138a0b90e262296bf8f 2024-11-13T00:14:43,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/439fcdc23bbd41ba934477effe52a614 is 50, key is test_row_0/C:col10/1731456883938/Put/seqid=0 2024-11-13T00:14:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742136_1312 (size=12001) 2024-11-13T00:14:43,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/439fcdc23bbd41ba934477effe52a614 2024-11-13T00:14:43,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9 2024-11-13T00:14:43,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9, entries=200, sequenceid=123, filesize=14.0 K 2024-11-13T00:14:43,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/0100d60289044138a0b90e262296bf8f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/0100d60289044138a0b90e262296bf8f 2024-11-13T00:14:43,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/0100d60289044138a0b90e262296bf8f, entries=150, sequenceid=123, filesize=11.7 K 2024-11-13T00:14:43,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/439fcdc23bbd41ba934477effe52a614 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/439fcdc23bbd41ba934477effe52a614 2024-11-13T00:14:44,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/439fcdc23bbd41ba934477effe52a614, entries=150, sequenceid=123, filesize=11.7 K 2024-11-13T00:14:44,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for dc9547c78a88f5a8e277773a4b75b850 in 63ms, sequenceid=123, compaction requested=true 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:44,001 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:44,001 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:44,002 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:44,002 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:44,002 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:44,002 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:44,002 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:44,002 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:44,002 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/8834835a36ce45969c0e96451cb9e491, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/435d1bde48ef41b0a25ba24b7494081c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/0100d60289044138a0b90e262296bf8f] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.4 K 2024-11-13T00:14:44,002 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/712e5a09341a4da78d9e2ab546bfa5a6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ced4f2979e9345c98511bbcb2a1a03d1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=37.6 K 2024-11-13T00:14:44,004 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8834835a36ce45969c0e96451cb9e491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731456880913 2024-11-13T00:14:44,004 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712e5a09341a4da78d9e2ab546bfa5a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731456880913 2024-11-13T00:14:44,004 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 435d1bde48ef41b0a25ba24b7494081c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731456881630 2024-11-13T00:14:44,004 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ced4f2979e9345c98511bbcb2a1a03d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731456881630 2024-11-13T00:14:44,005 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f0f8eb0c3bd467cb66b7496b4d1c1a9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731456882798 2024-11-13T00:14:44,005 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0100d60289044138a0b90e262296bf8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731456882798 2024-11-13T00:14:44,012 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:44,013 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/84e98859c37d47a5ad2ab606381ab96b is 50, key is test_row_0/A:col10/1731456883938/Put/seqid=0 2024-11-13T00:14:44,015 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:44,015 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/b15f6f6ee1b94fd88b99e05ff45249a9 is 50, key is test_row_0/B:col10/1731456883938/Put/seqid=0 2024-11-13T00:14:44,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742137_1313 (size=12309) 2024-11-13T00:14:44,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742138_1314 (size=12309) 2024-11-13T00:14:44,023 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/84e98859c37d47a5ad2ab606381ab96b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/84e98859c37d47a5ad2ab606381ab96b 2024-11-13T00:14:44,026 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/b15f6f6ee1b94fd88b99e05ff45249a9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b15f6f6ee1b94fd88b99e05ff45249a9 2024-11-13T00:14:44,030 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 84e98859c37d47a5ad2ab606381ab96b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:44,030 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:44,030 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=13, startTime=1731456884001; duration=0sec 2024-11-13T00:14:44,030 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:44,030 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:44,030 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:44,032 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:44,032 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:44,032 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into b15f6f6ee1b94fd88b99e05ff45249a9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:44,032 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:44,032 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/64ecb204bbef45ee8a1b1e12dddaf874, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/82a132bc78074c549632847852eea940, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/439fcdc23bbd41ba934477effe52a614] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=35.4 K 2024-11-13T00:14:44,032 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:44,033 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=13, startTime=1731456884001; duration=0sec 2024-11-13T00:14:44,033 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:44,033 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:44,033 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ecb204bbef45ee8a1b1e12dddaf874, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731456880913 2024-11-13T00:14:44,033 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a132bc78074c549632847852eea940, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731456881630 2024-11-13T00:14:44,034 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439fcdc23bbd41ba934477effe52a614, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731456882798 2024-11-13T00:14:44,041 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:44,042 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b is 50, key is test_row_0/C:col10/1731456883938/Put/seqid=0 2024-11-13T00:14:44,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742139_1315 (size=12309) 2024-11-13T00:14:44,053 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b 2024-11-13T00:14:44,059 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into b07e6431f0b04b7f9a3bc0d47f3e4c2b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:44,059 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:44,059 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=13, startTime=1731456884001; duration=0sec 2024-11-13T00:14:44,059 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:44,059 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:44,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:44,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/536e83fbec3b472781c6a2dd26f1540e is 50, key is test_row_0/A:col10/1731456884067/Put/seqid=0 2024-11-13T00:14:44,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742140_1316 (size=12151) 2024-11-13T00:14:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456944135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456944138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456944139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456944138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456944246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456944256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456944256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456944256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456944456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456944466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456944466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456944466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/536e83fbec3b472781c6a2dd26f1540e 2024-11-13T00:14:44,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/50f082c571b4468285ed19c01c978fc1 is 50, key is test_row_0/B:col10/1731456884067/Put/seqid=0 2024-11-13T00:14:44,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742141_1317 (size=12151) 2024-11-13T00:14:44,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456944766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456944776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456944776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456944777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:44,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/50f082c571b4468285ed19c01c978fc1 2024-11-13T00:14:44,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8a64546fb2304642a5f60430119b88da is 50, key is test_row_0/C:col10/1731456884067/Put/seqid=0 2024-11-13T00:14:44,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742142_1318 (size=12151) 2024-11-13T00:14:44,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:44,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456944966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:44,968 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in dc9547c78a88f5a8e277773a4b75b850 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456945276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:45,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:45,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456945286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456945286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456945286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:45,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8a64546fb2304642a5f60430119b88da 2024-11-13T00:14:45,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/536e83fbec3b472781c6a2dd26f1540e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/536e83fbec3b472781c6a2dd26f1540e 2024-11-13T00:14:45,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/536e83fbec3b472781c6a2dd26f1540e, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T00:14:45,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/50f082c571b4468285ed19c01c978fc1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/50f082c571b4468285ed19c01c978fc1 2024-11-13T00:14:45,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/50f082c571b4468285ed19c01c978fc1, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T00:14:45,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8a64546fb2304642a5f60430119b88da as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a64546fb2304642a5f60430119b88da 2024-11-13T00:14:45,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a64546fb2304642a5f60430119b88da, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T00:14:45,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for dc9547c78a88f5a8e277773a4b75b850 in 1308ms, sequenceid=142, compaction requested=false 2024-11-13T00:14:45,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T00:14:45,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:45,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T00:14:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T00:14:45,859 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:45,860 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:45,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T00:14:46,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:46,013 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:46,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/5b6956a49c7e4f1f83730ef549b46ffe is 50, key is test_row_0/A:col10/1731456884133/Put/seqid=0 2024-11-13T00:14:46,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742143_1319 (size=12151) 2024-11-13T00:14:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T00:14:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:46,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456946316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456946317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456946319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456946319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,423 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/5b6956a49c7e4f1f83730ef549b46ffe 2024-11-13T00:14:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456946426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456946426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5c2f034873b6477aba30c241be5e3752 is 50, key is test_row_0/B:col10/1731456884133/Put/seqid=0 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456946426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456946428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742144_1320 (size=12151) 2024-11-13T00:14:46,437 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5c2f034873b6477aba30c241be5e3752 2024-11-13T00:14:46,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/52ee682ce4804609bb02348cf963beaa is 50, key is test_row_0/C:col10/1731456884133/Put/seqid=0 2024-11-13T00:14:46,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742145_1321 (size=12151) 2024-11-13T00:14:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T00:14:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456946636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456946636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456946636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456946636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,848 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/52ee682ce4804609bb02348cf963beaa 2024-11-13T00:14:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/5b6956a49c7e4f1f83730ef549b46ffe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/5b6956a49c7e4f1f83730ef549b46ffe 2024-11-13T00:14:46,857 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/5b6956a49c7e4f1f83730ef549b46ffe, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T00:14:46,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5c2f034873b6477aba30c241be5e3752 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5c2f034873b6477aba30c241be5e3752 2024-11-13T00:14:46,861 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5c2f034873b6477aba30c241be5e3752, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T00:14:46,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/52ee682ce4804609bb02348cf963beaa as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/52ee682ce4804609bb02348cf963beaa 2024-11-13T00:14:46,865 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/52ee682ce4804609bb02348cf963beaa, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T00:14:46,866 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for dc9547c78a88f5a8e277773a4b75b850 in 853ms, sequenceid=158, compaction requested=true 2024-11-13T00:14:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T00:14:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T00:14:46,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T00:14:46,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-13T00:14:46,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0150 sec 2024-11-13T00:14:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:46,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T00:14:46,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:46,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:46,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/f7e0b28ea4a443d5a3db3188edf938fd is 50, key is test_row_0/A:col10/1731456886950/Put/seqid=0 2024-11-13T00:14:46,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456946970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456946972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742146_1322 (size=14541) 2024-11-13T00:14:46,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/f7e0b28ea4a443d5a3db3188edf938fd 2024-11-13T00:14:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456946976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456946977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:46,983 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in dc9547c78a88f5a8e277773a4b75b850 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456946977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:46,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/135cc02e0c98428fb6a06fe986d3a63c is 50, key is test_row_0/B:col10/1731456886950/Put/seqid=0 2024-11-13T00:14:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T00:14:46,996 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:46,996 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T00:14:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T00:14:46,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:46,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:46,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:47,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742147_1323 (size=12151) 2024-11-13T00:14:47,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/135cc02e0c98428fb6a06fe986d3a63c 2024-11-13T00:14:47,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e622cb53789d489f95342b88825c0b1c is 50, key is test_row_0/C:col10/1731456886950/Put/seqid=0 2024-11-13T00:14:47,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742148_1324 (size=12151) 2024-11-13T00:14:47,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e622cb53789d489f95342b88825c0b1c 2024-11-13T00:14:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/f7e0b28ea4a443d5a3db3188edf938fd as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f7e0b28ea4a443d5a3db3188edf938fd 2024-11-13T00:14:47,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f7e0b28ea4a443d5a3db3188edf938fd, entries=200, sequenceid=177, filesize=14.2 K 2024-11-13T00:14:47,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/135cc02e0c98428fb6a06fe986d3a63c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/135cc02e0c98428fb6a06fe986d3a63c 2024-11-13T00:14:47,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/135cc02e0c98428fb6a06fe986d3a63c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T00:14:47,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/e622cb53789d489f95342b88825c0b1c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e622cb53789d489f95342b88825c0b1c 2024-11-13T00:14:47,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e622cb53789d489f95342b88825c0b1c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T00:14:47,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for dc9547c78a88f5a8e277773a4b75b850 in 138ms, sequenceid=177, compaction requested=true 2024-11-13T00:14:47,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:47,089 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:47,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:47,091 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:47,091 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:47,091 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:47,091 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,092 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/84e98859c37d47a5ad2ab606381ab96b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/536e83fbec3b472781c6a2dd26f1540e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/5b6956a49c7e4f1f83730ef549b46ffe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f7e0b28ea4a443d5a3db3188edf938fd] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=50.0 K 2024-11-13T00:14:47,092 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e98859c37d47a5ad2ab606381ab96b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731456882798 2024-11-13T00:14:47,092 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 536e83fbec3b472781c6a2dd26f1540e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456884067 2024-11-13T00:14:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:47,093 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:47,093 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:47,093 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,093 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b15f6f6ee1b94fd88b99e05ff45249a9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/50f082c571b4468285ed19c01c978fc1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5c2f034873b6477aba30c241be5e3752, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/135cc02e0c98428fb6a06fe986d3a63c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=47.6 K 2024-11-13T00:14:47,094 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6956a49c7e4f1f83730ef549b46ffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731456884133 2024-11-13T00:14:47,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T00:14:47,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:47,094 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b15f6f6ee1b94fd88b99e05ff45249a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731456882798 2024-11-13T00:14:47,095 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7e0b28ea4a443d5a3db3188edf938fd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731456886316 2024-11-13T00:14:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:47,096 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 50f082c571b4468285ed19c01c978fc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456884067 2024-11-13T00:14:47,096 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2f034873b6477aba30c241be5e3752, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731456884133 2024-11-13T00:14:47,098 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 135cc02e0c98428fb6a06fe986d3a63c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731456886316 2024-11-13T00:14:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T00:14:47,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/f4afe67772f248af96ef58821663806c is 50, key is test_row_0/A:col10/1731456887090/Put/seqid=0 2024-11-13T00:14:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456947120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456947121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456947122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456947130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,144 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#271 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:47,146 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#272 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:47,146 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e727e20b20df4114ac9117a2c7322116 is 50, key is test_row_0/B:col10/1731456886950/Put/seqid=0 2024-11-13T00:14:47,146 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/ac5ce6ee6eab43d48041ed6e1e935cff is 50, key is test_row_0/A:col10/1731456886950/Put/seqid=0 2024-11-13T00:14:47,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742149_1325 (size=12151) 2024-11-13T00:14:47,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/f4afe67772f248af96ef58821663806c 2024-11-13T00:14:47,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742150_1326 (size=12595) 2024-11-13T00:14:47,185 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/ac5ce6ee6eab43d48041ed6e1e935cff as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ac5ce6ee6eab43d48041ed6e1e935cff 2024-11-13T00:14:47,190 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into ac5ce6ee6eab43d48041ed6e1e935cff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:47,190 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:47,190 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=12, startTime=1731456887089; duration=0sec 2024-11-13T00:14:47,191 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:47,191 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:47,191 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:47,192 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:47,192 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:47,192 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,192 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a64546fb2304642a5f60430119b88da, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/52ee682ce4804609bb02348cf963beaa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e622cb53789d489f95342b88825c0b1c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=47.6 K 2024-11-13T00:14:47,193 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b07e6431f0b04b7f9a3bc0d47f3e4c2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731456882798 2024-11-13T00:14:47,194 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a64546fb2304642a5f60430119b88da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731456884067 2024-11-13T00:14:47,194 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ee682ce4804609bb02348cf963beaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731456884133 2024-11-13T00:14:47,194 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e622cb53789d489f95342b88825c0b1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731456886316 2024-11-13T00:14:47,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742151_1327 (size=12595) 2024-11-13T00:14:47,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e5552657195a47d69b752b668f8625e0 is 50, key is test_row_0/B:col10/1731456887090/Put/seqid=0 2024-11-13T00:14:47,218 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:47,219 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/1331c85093794f24bfab7260c610460a is 50, key is test_row_0/C:col10/1731456886950/Put/seqid=0 2024-11-13T00:14:47,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742152_1328 (size=12151) 2024-11-13T00:14:47,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742153_1329 (size=12595) 2024-11-13T00:14:47,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456947237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456947237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456947237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456947246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T00:14:47,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456947446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456947446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456947446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:47,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:47,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456947456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,611 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e727e20b20df4114ac9117a2c7322116 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e727e20b20df4114ac9117a2c7322116 2024-11-13T00:14:47,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:47,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,617 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into e727e20b20df4114ac9117a2c7322116(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:47,617 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:47,617 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=12, startTime=1731456887091; duration=0sec 2024-11-13T00:14:47,617 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:47,617 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T00:14:47,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e5552657195a47d69b752b668f8625e0 2024-11-13T00:14:47,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/93617abd70bf4fcd887e2144d37506ea is 50, key is test_row_0/C:col10/1731456887090/Put/seqid=0 2024-11-13T00:14:47,642 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/1331c85093794f24bfab7260c610460a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/1331c85093794f24bfab7260c610460a 2024-11-13T00:14:47,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742154_1330 (size=12151) 2024-11-13T00:14:47,648 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 1331c85093794f24bfab7260c610460a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:47,648 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:47,648 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=12, startTime=1731456887091; duration=0sec 2024-11-13T00:14:47,649 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:47,649 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456947756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456947756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456947756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456947765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:47,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:47,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:48,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/93617abd70bf4fcd887e2144d37506ea 2024-11-13T00:14:48,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/f4afe67772f248af96ef58821663806c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f4afe67772f248af96ef58821663806c 2024-11-13T00:14:48,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f4afe67772f248af96ef58821663806c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T00:14:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/e5552657195a47d69b752b668f8625e0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5552657195a47d69b752b668f8625e0 2024-11-13T00:14:48,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5552657195a47d69b752b668f8625e0, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T00:14:48,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/93617abd70bf4fcd887e2144d37506ea as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/93617abd70bf4fcd887e2144d37506ea 2024-11-13T00:14:48,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/93617abd70bf4fcd887e2144d37506ea, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T00:14:48,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for dc9547c78a88f5a8e277773a4b75b850 in 972ms, sequenceid=193, compaction requested=false 2024-11-13T00:14:48,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:48,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:48,071 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:48,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/58d42d63eec240d8af5adf3805968462 is 50, key is test_row_0/A:col10/1731456887118/Put/seqid=0 2024-11-13T00:14:48,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742155_1331 (size=12151) 2024-11-13T00:14:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T00:14:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:48,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:48,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456948287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456948289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456948290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456948290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456948396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456948396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456948406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456948406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,480 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/58d42d63eec240d8af5adf3805968462 2024-11-13T00:14:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/1a88786eac46400bbbeda7c4a544305c is 50, key is test_row_0/B:col10/1731456887118/Put/seqid=0 2024-11-13T00:14:48,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742156_1332 (size=12151) 2024-11-13T00:14:48,492 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/1a88786eac46400bbbeda7c4a544305c 2024-11-13T00:14:48,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9df0477ad9cd46ba88b66052ca69d844 is 50, key is test_row_0/C:col10/1731456887118/Put/seqid=0 2024-11-13T00:14:48,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742157_1333 (size=12151) 2024-11-13T00:14:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456948606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456948606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456948616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456948616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,903 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9df0477ad9cd46ba88b66052ca69d844 2024-11-13T00:14:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/58d42d63eec240d8af5adf3805968462 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/58d42d63eec240d8af5adf3805968462 2024-11-13T00:14:48,913 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/58d42d63eec240d8af5adf3805968462, entries=150, sequenceid=212, filesize=11.9 K 2024-11-13T00:14:48,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/1a88786eac46400bbbeda7c4a544305c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a88786eac46400bbbeda7c4a544305c 2024-11-13T00:14:48,918 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a88786eac46400bbbeda7c4a544305c, entries=150, sequenceid=212, filesize=11.9 K 2024-11-13T00:14:48,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456948916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456948916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9df0477ad9cd46ba88b66052ca69d844 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9df0477ad9cd46ba88b66052ca69d844 2024-11-13T00:14:48,923 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9df0477ad9cd46ba88b66052ca69d844, entries=150, sequenceid=212, filesize=11.9 K 2024-11-13T00:14:48,924 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for dc9547c78a88f5a8e277773a4b75b850 in 853ms, sequenceid=212, compaction requested=true 2024-11-13T00:14:48,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:48,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:48,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T00:14:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T00:14:48,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T00:14:48,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9260 sec 2024-11-13T00:14:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:48,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T00:14:48,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:48,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:48,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:48,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:48,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:48,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:48,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.9320 sec 2024-11-13T00:14:48,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/e93cd77e9647477eb171dc9998ed6f84 is 50, key is test_row_0/A:col10/1731456888288/Put/seqid=0 2024-11-13T00:14:48,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742158_1334 (size=14541) 2024-11-13T00:14:48,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/e93cd77e9647477eb171dc9998ed6f84 2024-11-13T00:14:48,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/1a7fd697295143699c15a3f7fb076106 is 50, key is test_row_0/B:col10/1731456888288/Put/seqid=0 2024-11-13T00:14:48,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742159_1335 (size=12151) 2024-11-13T00:14:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456948957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456948959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456949066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456949066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T00:14:49,146 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:49,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-13T00:14:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T00:14:49,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:49,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:49,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T00:14:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456949276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456949276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T00:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/1a7fd697295143699c15a3f7fb076106 2024-11-13T00:14:49,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/054f3295c6c04eddbc583d5f0ecc5b8c is 50, key is test_row_0/C:col10/1731456888288/Put/seqid=0 2024-11-13T00:14:49,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742160_1336 (size=12151) 2024-11-13T00:14:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456949426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456949426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T00:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T00:14:49,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456949586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:49,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456949586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:49,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:49,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:49,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T00:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:49,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T00:14:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:49,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/054f3295c6c04eddbc583d5f0ecc5b8c 2024-11-13T00:14:49,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/e93cd77e9647477eb171dc9998ed6f84 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e93cd77e9647477eb171dc9998ed6f84 2024-11-13T00:14:49,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e93cd77e9647477eb171dc9998ed6f84, entries=200, sequenceid=230, filesize=14.2 K 2024-11-13T00:14:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/1a7fd697295143699c15a3f7fb076106 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a7fd697295143699c15a3f7fb076106 2024-11-13T00:14:49,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a7fd697295143699c15a3f7fb076106, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T00:14:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T00:14:49,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/054f3295c6c04eddbc583d5f0ecc5b8c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/054f3295c6c04eddbc583d5f0ecc5b8c 2024-11-13T00:14:49,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/054f3295c6c04eddbc583d5f0ecc5b8c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T00:14:49,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for dc9547c78a88f5a8e277773a4b75b850 in 853ms, sequenceid=230, compaction requested=true 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:49,782 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:49,782 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:49,783 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:49,783 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:49,784 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,784 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ac5ce6ee6eab43d48041ed6e1e935cff, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f4afe67772f248af96ef58821663806c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/58d42d63eec240d8af5adf3805968462, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e93cd77e9647477eb171dc9998ed6f84] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=50.2 K 2024-11-13T00:14:49,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:49,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:49,784 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,784 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e727e20b20df4114ac9117a2c7322116, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5552657195a47d69b752b668f8625e0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a88786eac46400bbbeda7c4a544305c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a7fd697295143699c15a3f7fb076106] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=47.9 K 2024-11-13T00:14:49,784 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5ce6ee6eab43d48041ed6e1e935cff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731456886316 2024-11-13T00:14:49,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e727e20b20df4114ac9117a2c7322116, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731456886316 2024-11-13T00:14:49,785 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5552657195a47d69b752b668f8625e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731456886958 2024-11-13T00:14:49,785 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f4afe67772f248af96ef58821663806c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731456886958 2024-11-13T00:14:49,785 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a88786eac46400bbbeda7c4a544305c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731456887099 2024-11-13T00:14:49,785 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d42d63eec240d8af5adf3805968462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731456887099 2024-11-13T00:14:49,785 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7fd697295143699c15a3f7fb076106, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731456888280 2024-11-13T00:14:49,786 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e93cd77e9647477eb171dc9998ed6f84, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731456888280 2024-11-13T00:14:49,796 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#283 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:49,796 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#282 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:49,796 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/6ca758c54334443da13cbe9dec6f1f40 is 50, key is test_row_0/B:col10/1731456888288/Put/seqid=0 2024-11-13T00:14:49,797 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9f67bb64a4f047369736109330a7e4a6 is 50, key is test_row_0/A:col10/1731456888288/Put/seqid=0 2024-11-13T00:14:49,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742161_1337 (size=12731) 2024-11-13T00:14:49,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742162_1338 (size=12731) 2024-11-13T00:14:49,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T00:14:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:49,916 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:14:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/0721fa8ef5fc436ba061165023d62348 is 50, key is test_row_0/A:col10/1731456888958/Put/seqid=0 2024-11-13T00:14:49,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742163_1339 (size=12151) 2024-11-13T00:14:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:50,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456950165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456950165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,207 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/6ca758c54334443da13cbe9dec6f1f40 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/6ca758c54334443da13cbe9dec6f1f40 2024-11-13T00:14:50,212 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into 6ca758c54334443da13cbe9dec6f1f40(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:50,212 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:50,212 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=12, startTime=1731456889782; duration=0sec 2024-11-13T00:14:50,212 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:50,212 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:50,212 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:50,217 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:50,217 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:50,217 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:50,217 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/1331c85093794f24bfab7260c610460a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/93617abd70bf4fcd887e2144d37506ea, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9df0477ad9cd46ba88b66052ca69d844, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/054f3295c6c04eddbc583d5f0ecc5b8c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=47.9 K 2024-11-13T00:14:50,217 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1331c85093794f24bfab7260c610460a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731456886316 2024-11-13T00:14:50,218 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93617abd70bf4fcd887e2144d37506ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731456886958 2024-11-13T00:14:50,221 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9f67bb64a4f047369736109330a7e4a6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f67bb64a4f047369736109330a7e4a6 2024-11-13T00:14:50,221 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df0477ad9cd46ba88b66052ca69d844, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731456887099 2024-11-13T00:14:50,222 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054f3295c6c04eddbc583d5f0ecc5b8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731456888280 2024-11-13T00:14:50,227 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 9f67bb64a4f047369736109330a7e4a6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:50,227 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:50,227 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=12, startTime=1731456889782; duration=0sec 2024-11-13T00:14:50,227 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:50,227 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:50,234 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:50,235 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9e6a3f7e85694edbb0ac6fb4531d4d6d is 50, key is test_row_0/C:col10/1731456888288/Put/seqid=0 2024-11-13T00:14:50,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742164_1340 (size=12731) 2024-11-13T00:14:50,251 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9e6a3f7e85694edbb0ac6fb4531d4d6d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9e6a3f7e85694edbb0ac6fb4531d4d6d 2024-11-13T00:14:50,255 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 9e6a3f7e85694edbb0ac6fb4531d4d6d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:50,255 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:50,256 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=12, startTime=1731456889782; duration=0sec 2024-11-13T00:14:50,256 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:50,256 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456950276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456950276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T00:14:50,326 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/0721fa8ef5fc436ba061165023d62348 2024-11-13T00:14:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5ccccd7d6db94e51ac51edd021fd3f2c is 50, key is test_row_0/B:col10/1731456888958/Put/seqid=0 2024-11-13T00:14:50,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742165_1341 (size=12151) 2024-11-13T00:14:50,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456950436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,437 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in dc9547c78a88f5a8e277773a4b75b850 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456950446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,449 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in dc9547c78a88f5a8e277773a4b75b850 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456950486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456950486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,738 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5ccccd7d6db94e51ac51edd021fd3f2c 2024-11-13T00:14:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/292ca81eacfb49efa0d7673ba5616ec7 is 50, key is test_row_0/C:col10/1731456888958/Put/seqid=0 2024-11-13T00:14:50,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742166_1342 (size=12151) 2024-11-13T00:14:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456950795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456950796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:50,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731456950996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:50,999 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in dc9547c78a88f5a8e277773a4b75b850 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:14:51,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,150 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/292ca81eacfb49efa0d7673ba5616ec7 2024-11-13T00:14:51,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/0721fa8ef5fc436ba061165023d62348 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/0721fa8ef5fc436ba061165023d62348 2024-11-13T00:14:51,158 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/0721fa8ef5fc436ba061165023d62348, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T00:14:51,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5ccccd7d6db94e51ac51edd021fd3f2c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5ccccd7d6db94e51ac51edd021fd3f2c 2024-11-13T00:14:51,163 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5ccccd7d6db94e51ac51edd021fd3f2c, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T00:14:51,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/292ca81eacfb49efa0d7673ba5616ec7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/292ca81eacfb49efa0d7673ba5616ec7 2024-11-13T00:14:51,168 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/292ca81eacfb49efa0d7673ba5616ec7, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T00:14:51,169 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for dc9547c78a88f5a8e277773a4b75b850 in 1253ms, sequenceid=244, compaction requested=false 2024-11-13T00:14:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-13T00:14:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-13T00:14:51,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T00:14:51,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-11-13T00:14:51,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.0280 sec 2024-11-13T00:14:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T00:14:51,297 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:51,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-13T00:14:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T00:14:51,299 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:51,300 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:51,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:51,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T00:14:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:51,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/8629eac2952d4817b27c38baf56e1e65 is 50, key is test_row_0/A:col10/1731456891308/Put/seqid=0 2024-11-13T00:14:51,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742167_1343 (size=12301) 2024-11-13T00:14:51,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456951329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456951332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T00:14:51,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456951446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456951446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T00:14:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T00:14:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T00:14:51,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456951656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456951656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/8629eac2952d4817b27c38baf56e1e65 2024-11-13T00:14:51,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/cff6381f4873478da05abe6df9829b26 is 50, key is test_row_0/B:col10/1731456891308/Put/seqid=0 2024-11-13T00:14:51,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742168_1344 (size=12301) 2024-11-13T00:14:51,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T00:14:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T00:14:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:51,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T00:14:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456951966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456951966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T00:14:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:52,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:52,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/cff6381f4873478da05abe6df9829b26 2024-11-13T00:14:52,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/172c5929948d4bb4ba7628852f4191fb is 50, key is test_row_0/C:col10/1731456891308/Put/seqid=0 2024-11-13T00:14:52,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742169_1345 (size=12301) 2024-11-13T00:14:52,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/172c5929948d4bb4ba7628852f4191fb 2024-11-13T00:14:52,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/8629eac2952d4817b27c38baf56e1e65 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/8629eac2952d4817b27c38baf56e1e65 2024-11-13T00:14:52,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/8629eac2952d4817b27c38baf56e1e65, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T00:14:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/cff6381f4873478da05abe6df9829b26 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/cff6381f4873478da05abe6df9829b26 2024-11-13T00:14:52,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/cff6381f4873478da05abe6df9829b26, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T00:14:52,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/172c5929948d4bb4ba7628852f4191fb as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/172c5929948d4bb4ba7628852f4191fb 2024-11-13T00:14:52,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/172c5929948d4bb4ba7628852f4191fb, entries=150, sequenceid=268, filesize=12.0 K 2024-11-13T00:14:52,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for dc9547c78a88f5a8e277773a4b75b850 in 870ms, sequenceid=268, compaction requested=true 2024-11-13T00:14:52,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:52,181 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:52,181 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:52,182 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:52,182 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:52,182 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:52,182 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f67bb64a4f047369736109330a7e4a6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/0721fa8ef5fc436ba061165023d62348, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/8629eac2952d4817b27c38baf56e1e65] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=36.3 K 2024-11-13T00:14:52,182 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:52,182 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:52,182 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:52,182 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/6ca758c54334443da13cbe9dec6f1f40, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5ccccd7d6db94e51ac51edd021fd3f2c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/cff6381f4873478da05abe6df9829b26] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=36.3 K 2024-11-13T00:14:52,182 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f67bb64a4f047369736109330a7e4a6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731456888280 2024-11-13T00:14:52,182 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca758c54334443da13cbe9dec6f1f40, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731456888280 2024-11-13T00:14:52,183 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0721fa8ef5fc436ba061165023d62348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731456888951 2024-11-13T00:14:52,183 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ccccd7d6db94e51ac51edd021fd3f2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731456888951 2024-11-13T00:14:52,183 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting cff6381f4873478da05abe6df9829b26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731456890140 2024-11-13T00:14:52,183 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8629eac2952d4817b27c38baf56e1e65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731456890140 2024-11-13T00:14:52,202 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:52,203 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a479b084db644762b870894bd183a0b8 is 50, key is test_row_0/B:col10/1731456891308/Put/seqid=0 2024-11-13T00:14:52,206 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:52,207 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/744426c36acc43978280d0bf79e7cda0 is 50, key is test_row_0/A:col10/1731456891308/Put/seqid=0 2024-11-13T00:14:52,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:52,218 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:52,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:52,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742170_1346 (size=12983) 2024-11-13T00:14:52,232 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/744426c36acc43978280d0bf79e7cda0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/744426c36acc43978280d0bf79e7cda0 2024-11-13T00:14:52,237 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 744426c36acc43978280d0bf79e7cda0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:52,237 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:52,237 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=13, startTime=1731456892180; duration=0sec 2024-11-13T00:14:52,237 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:52,237 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:52,237 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:52,238 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:52,238 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:52,238 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:52,238 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9e6a3f7e85694edbb0ac6fb4531d4d6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/292ca81eacfb49efa0d7673ba5616ec7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/172c5929948d4bb4ba7628852f4191fb] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=36.3 K 2024-11-13T00:14:52,239 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e6a3f7e85694edbb0ac6fb4531d4d6d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731456888280 2024-11-13T00:14:52,239 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 292ca81eacfb49efa0d7673ba5616ec7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731456888951 2024-11-13T00:14:52,240 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172c5929948d4bb4ba7628852f4191fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731456890140 2024-11-13T00:14:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a1faeefab7c040b7b9efcc7585877c6d is 50, key is test_row_0/A:col10/1731456891327/Put/seqid=0 2024-11-13T00:14:52,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742171_1347 (size=12983) 2024-11-13T00:14:52,258 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:52,259 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9ca3ca14f0314f67a0a4d110741fd112 is 50, key is test_row_0/C:col10/1731456891308/Put/seqid=0 2024-11-13T00:14:52,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742172_1348 (size=12301) 2024-11-13T00:14:52,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742173_1349 (size=12983) 2024-11-13T00:14:52,268 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/9ca3ca14f0314f67a0a4d110741fd112 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9ca3ca14f0314f67a0a4d110741fd112 2024-11-13T00:14:52,273 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 9ca3ca14f0314f67a0a4d110741fd112(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:52,273 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:52,273 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=13, startTime=1731456892181; duration=0sec 2024-11-13T00:14:52,273 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:52,273 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T00:14:52,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:52,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456892501, max timestamp=1731456892529Num uniques 859; 2024-11-13T00:14:52,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:52,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456892501, max timestamp=1731456892531Num uniques 859; 2024-11-13T00:14:52,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:52,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456892501, max timestamp=1731456892529Num uniques 859; 2024-11-13T00:14:52,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456952577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456952577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456952579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456952588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,660 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a479b084db644762b870894bd183a0b8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a479b084db644762b870894bd183a0b8 2024-11-13T00:14:52,662 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a1faeefab7c040b7b9efcc7585877c6d 2024-11-13T00:14:52,669 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into a479b084db644762b870894bd183a0b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:52,669 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:52,669 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=13, startTime=1731456892181; duration=0sec 2024-11-13T00:14:52,669 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:52,669 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ae65362331ea4fd899bc5f6af4d8ae06 is 50, key is test_row_0/B:col10/1731456891327/Put/seqid=0 2024-11-13T00:14:52,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742174_1350 (size=12301) 2024-11-13T00:14:52,691 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ae65362331ea4fd899bc5f6af4d8ae06 2024-11-13T00:14:52,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456952696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456952697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456952697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/40d7f1b8350d46b69fe4ab274e3401a3 is 50, key is test_row_0/C:col10/1731456891327/Put/seqid=0 2024-11-13T00:14:52,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456952706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742175_1351 (size=12301) 2024-11-13T00:14:52,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:14:52,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456952906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456952906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456952907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:52,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456952916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,119 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/40d7f1b8350d46b69fe4ab274e3401a3 2024-11-13T00:14:53,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a1faeefab7c040b7b9efcc7585877c6d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1faeefab7c040b7b9efcc7585877c6d 2024-11-13T00:14:53,127 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1faeefab7c040b7b9efcc7585877c6d, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T00:14:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ae65362331ea4fd899bc5f6af4d8ae06 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ae65362331ea4fd899bc5f6af4d8ae06 2024-11-13T00:14:53,132 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ae65362331ea4fd899bc5f6af4d8ae06, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T00:14:53,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/40d7f1b8350d46b69fe4ab274e3401a3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/40d7f1b8350d46b69fe4ab274e3401a3 2024-11-13T00:14:53,136 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/40d7f1b8350d46b69fe4ab274e3401a3, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T00:14:53,137 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for dc9547c78a88f5a8e277773a4b75b850 in 919ms, sequenceid=279, compaction requested=false 2024-11-13T00:14:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-13T00:14:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-13T00:14:53,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T00:14:53,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8390 sec 2024-11-13T00:14:53,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8440 sec 2024-11-13T00:14:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:53,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=143.13 KB heapSize=410.69 KB 2024-11-13T00:14:53,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456892574, max timestamp=1731456893221Num uniques 1018; 2024-11-13T00:14:53,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T00:14:53,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456892574, max timestamp=1731456893221Num uniques 1068; 2024-11-13T00:14:53,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T00:14:53,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456892577, max timestamp=1731456893221Num uniques 1018; 2024-11-13T00:14:53,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:14:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:53,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:53,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/7c77aae6bff5405ca801ce2fa174f131 is 50, key is test_row_0/A:col10/1731456892585/Put/seqid=0 2024-11-13T00:14:53,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742176_1352 (size=14741) 2024-11-13T00:14:53,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/7c77aae6bff5405ca801ce2fa174f131 2024-11-13T00:14:53,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/58e193c0b5bb4ac5b002f63a96ab3be3 is 50, key is test_row_0/B:col10/1731456892585/Put/seqid=0 2024-11-13T00:14:53,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742177_1353 (size=12301) 2024-11-13T00:14:53,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/58e193c0b5bb4ac5b002f63a96ab3be3 2024-11-13T00:14:53,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456893252, max timestamp=1731456893261Num uniques 859; 2024-11-13T00:14:53,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456893252, max timestamp=1731456893261Num uniques 859; 2024-11-13T00:14:53,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456893252, max timestamp=1731456893261Num uniques 859; 2024-11-13T00:14:53,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:53,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:53,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:14:53,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/830a5d136420480380f5e4901f3c1a36 is 50, key is test_row_0/C:col10/1731456892585/Put/seqid=0 2024-11-13T00:14:53,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742178_1354 (size=12301) 2024-11-13T00:14:53,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731456893280, max timestamp=1731456893292Num uniques 1068; 2024-11-13T00:14:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456953289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T00:14:53,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731456893280, max timestamp=1731456893292Num uniques 1068; 2024-11-13T00:14:53,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T00:14:53,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731456893280, max timestamp=1731456893292Num uniques 1068; 2024-11-13T00:14:53,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-13T00:14:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456953337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731456893311, max timestamp=1731456893341Num uniques 1127; 2024-11-13T00:14:53,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731456893311, max timestamp=1731456893341Num uniques 1127; 2024-11-13T00:14:53,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T00:14:53,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731456893311, max timestamp=1731456893341Num uniques 1127; 2024-11-13T00:14:53,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T00:14:53,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-13T00:14:53,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456953395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731456893384, max timestamp=1731456893397Num uniques 1086; 2024-11-13T00:14:53,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731456893384, max timestamp=1731456893397Num uniques 1086; 2024-11-13T00:14:53,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T00:14:53,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T00:14:53,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731456893384, max timestamp=1731456893397Num uniques 1086; 2024-11-13T00:14:53,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-13T00:14:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T00:14:53,446 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:53,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-13T00:14:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:53,449 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:53,450 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:53,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:53,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731456893434, max timestamp=1731456893458Num uniques 1045; 2024-11-13T00:14:53,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731456893434, max timestamp=1731456893458Num uniques 1045; 2024-11-13T00:14:53,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-13T00:14:53,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-13T00:14:53,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731456893434, max timestamp=1731456893458Num uniques 1045; 2024-11-13T00:14:53,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-13T00:14:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456953499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456953503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456953505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456953506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:53,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456953616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456953616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456953616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/830a5d136420480380f5e4901f3c1a36 2024-11-13T00:14:53,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/7c77aae6bff5405ca801ce2fa174f131 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7c77aae6bff5405ca801ce2fa174f131 2024-11-13T00:14:53,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7c77aae6bff5405ca801ce2fa174f131, entries=200, sequenceid=321, filesize=14.4 K 2024-11-13T00:14:53,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/58e193c0b5bb4ac5b002f63a96ab3be3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/58e193c0b5bb4ac5b002f63a96ab3be3 2024-11-13T00:14:53,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/58e193c0b5bb4ac5b002f63a96ab3be3, entries=150, sequenceid=321, filesize=12.0 K 2024-11-13T00:14:53,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/830a5d136420480380f5e4901f3c1a36 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/830a5d136420480380f5e4901f3c1a36 2024-11-13T00:14:53,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/830a5d136420480380f5e4901f3c1a36, entries=150, sequenceid=321, filesize=12.0 K 2024-11-13T00:14:53,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~38.02 KB/38930, heapSize ~95.42 KB/97712, currentSize=154.31 KB/158010 for dc9547c78a88f5a8e277773a4b75b850 in 467ms, sequenceid=321, compaction requested=true 2024-11-13T00:14:53,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:53,692 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:53,692 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:53,699 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:53,699 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:53,699 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:53,699 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:53,699 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,699 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,699 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a479b084db644762b870894bd183a0b8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ae65362331ea4fd899bc5f6af4d8ae06, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/58e193c0b5bb4ac5b002f63a96ab3be3] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=36.7 K 2024-11-13T00:14:53,699 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/744426c36acc43978280d0bf79e7cda0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1faeefab7c040b7b9efcc7585877c6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7c77aae6bff5405ca801ce2fa174f131] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=39.1 K 2024-11-13T00:14:53,699 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744426c36acc43978280d0bf79e7cda0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731456890140 2024-11-13T00:14:53,699 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a479b084db644762b870894bd183a0b8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731456890140 2024-11-13T00:14:53,700 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1faeefab7c040b7b9efcc7585877c6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731456891318 2024-11-13T00:14:53,700 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ae65362331ea4fd899bc5f6af4d8ae06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731456891318 2024-11-13T00:14:53,700 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c77aae6bff5405ca801ce2fa174f131, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731456892585 2024-11-13T00:14:53,700 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e193c0b5bb4ac5b002f63a96ab3be3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731456892585 2024-11-13T00:14:53,707 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:53,708 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a955924e719f4248885e812afe021592 is 50, key is test_row_0/B:col10/1731456892585/Put/seqid=0 2024-11-13T00:14:53,715 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:53,715 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a25abcd3461a4f08b91fb7e6d2f64f2e is 50, key is test_row_0/A:col10/1731456892585/Put/seqid=0 2024-11-13T00:14:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:53,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=161.02 KB heapSize=452.63 KB 2024-11-13T00:14:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:14:53,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742179_1355 (size=13085) 2024-11-13T00:14:53,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/b74a948b9e2a48f2b65f8ab7e9f6c21d is 50, key is test_row_0/A:col10/1731456893504/Put/seqid=0 2024-11-13T00:14:53,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742180_1356 (size=13085) 2024-11-13T00:14:53,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742181_1357 (size=14741) 2024-11-13T00:14:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456953741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,748 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a25abcd3461a4f08b91fb7e6d2f64f2e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a25abcd3461a4f08b91fb7e6d2f64f2e 2024-11-13T00:14:53,752 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into a25abcd3461a4f08b91fb7e6d2f64f2e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:53,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:53,752 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=13, startTime=1731456893692; duration=0sec 2024-11-13T00:14:53,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:53,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:53,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:14:53,753 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:14:53,753 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:53,753 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,754 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9ca3ca14f0314f67a0a4d110741fd112, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/40d7f1b8350d46b69fe4ab274e3401a3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/830a5d136420480380f5e4901f3c1a36] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=36.7 K 2024-11-13T00:14:53,754 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca3ca14f0314f67a0a4d110741fd112, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731456890140 2024-11-13T00:14:53,755 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40d7f1b8350d46b69fe4ab274e3401a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731456891318 2024-11-13T00:14:53,755 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830a5d136420480380f5e4901f3c1a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731456892585 2024-11-13T00:14:53,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:53,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:53,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:53,777 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:53,777 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/24366c3d2b0942dc93fba4216623dfb6 is 50, key is test_row_0/C:col10/1731456892585/Put/seqid=0 2024-11-13T00:14:53,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742182_1358 (size=13085) 2024-11-13T00:14:53,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456953825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456953826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456953826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456953856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:53,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:53,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456954066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:54,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456954135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456954136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456954136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,142 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/a955924e719f4248885e812afe021592 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a955924e719f4248885e812afe021592 2024-11-13T00:14:54,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/b74a948b9e2a48f2b65f8ab7e9f6c21d 2024-11-13T00:14:54,147 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into a955924e719f4248885e812afe021592(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:54,147 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:54,147 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=13, startTime=1731456893692; duration=0sec 2024-11-13T00:14:54,147 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:54,147 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:54,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5cc9b9cfcd584eeca2eb419d684c8516 is 50, key is test_row_0/B:col10/1731456893504/Put/seqid=0 2024-11-13T00:14:54,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742183_1359 (size=12301) 2024-11-13T00:14:54,188 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/24366c3d2b0942dc93fba4216623dfb6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/24366c3d2b0942dc93fba4216623dfb6 2024-11-13T00:14:54,193 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 24366c3d2b0942dc93fba4216623dfb6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:54,193 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:54,193 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=13, startTime=1731456893692; duration=0sec 2024-11-13T00:14:54,193 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:54,193 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:54,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456954375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5cc9b9cfcd584eeca2eb419d684c8516 2024-11-13T00:14:54,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/73427ef4558a4824a5198254de152b67 is 50, key is test_row_0/C:col10/1731456893504/Put/seqid=0 2024-11-13T00:14:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742184_1360 (size=12301) 2024-11-13T00:14:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:54,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456954646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456954646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456954646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456954886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:54,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/73427ef4558a4824a5198254de152b67 2024-11-13T00:14:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/b74a948b9e2a48f2b65f8ab7e9f6c21d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/b74a948b9e2a48f2b65f8ab7e9f6c21d 2024-11-13T00:14:54,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/b74a948b9e2a48f2b65f8ab7e9f6c21d, entries=200, sequenceid=428, filesize=14.4 K 2024-11-13T00:14:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/5cc9b9cfcd584eeca2eb419d684c8516 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5cc9b9cfcd584eeca2eb419d684c8516 2024-11-13T00:14:54,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:54,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:54,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:54,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:54,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5cc9b9cfcd584eeca2eb419d684c8516, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T00:14:54,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/73427ef4558a4824a5198254de152b67 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/73427ef4558a4824a5198254de152b67 2024-11-13T00:14:54,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/73427ef4558a4824a5198254de152b67, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T00:14:54,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~452.58 KB/463440, currentSize=20.13 KB/20610 for dc9547c78a88f5a8e277773a4b75b850 in 1274ms, sequenceid=428, compaction requested=false 2024-11-13T00:14:54,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:55,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T00:14:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:55,137 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T00:14:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:55,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/2ecb996dd0ad44838c2549e0aa50de19 is 50, key is test_row_0/A:col10/1731456893739/Put/seqid=0 2024-11-13T00:14:55,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742185_1361 (size=9857) 2024-11-13T00:14:55,560 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/2ecb996dd0ad44838c2549e0aa50de19 2024-11-13T00:14:55,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/23ef91b521134e40b37a5188bcc1343e is 50, key is test_row_0/B:col10/1731456893739/Put/seqid=0 2024-11-13T00:14:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742186_1362 (size=9857) 2024-11-13T00:14:55,575 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/23ef91b521134e40b37a5188bcc1343e 2024-11-13T00:14:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8a279c4ce3304943856ad8515e4dd4bb is 50, key is test_row_0/C:col10/1731456893739/Put/seqid=0 2024-11-13T00:14:55,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742187_1363 (size=9857) 2024-11-13T00:14:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:55,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:55,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456895683, max timestamp=1731456895694Num uniques 859; 2024-11-13T00:14:55,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:55,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456895683, max timestamp=1731456895694Num uniques 859; 2024-11-13T00:14:55,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456895683, max timestamp=1731456895694Num uniques 859; 2024-11-13T00:14:55,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:55,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:14:55,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456895715, max timestamp=1731456895724Num uniques 1018; 2024-11-13T00:14:55,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:14:55,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456895715, max timestamp=1731456895724Num uniques 1018; 2024-11-13T00:14:55,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:14:55,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456895715, max timestamp=1731456895724Num uniques 1018; 2024-11-13T00:14:55,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:14:55,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731456895758, max timestamp=1731456895764Num uniques 1027; 2024-11-13T00:14:55,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T00:14:55,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731456895758, max timestamp=1731456895764Num uniques 1027; 2024-11-13T00:14:55,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731456895758, max timestamp=1731456895764Num uniques 1027; 2024-11-13T00:14:55,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T00:14:55,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T00:14:55,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731456895793, max timestamp=1731456895805Num uniques 1086; 2024-11-13T00:14:55,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T00:14:55,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731456895793, max timestamp=1731456895805Num uniques 1086; 2024-11-13T00:14:55,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T00:14:55,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731456895793, max timestamp=1731456895805Num uniques 1086; 2024-11-13T00:14:55,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-13T00:14:55,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731456895834, max timestamp=1731456895843Num uniques 1095; 2024-11-13T00:14:55,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731456895834, max timestamp=1731456895843Num uniques 1095; 2024-11-13T00:14:55,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T00:14:55,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T00:14:55,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731456895834, max timestamp=1731456895843Num uniques 1095; 2024-11-13T00:14:55,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-13T00:14:55,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731456895881, max timestamp=1731456895891Num uniques 1054; 2024-11-13T00:14:55,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731456895881, max timestamp=1731456895891Num uniques 1054; 2024-11-13T00:14:55,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-13T00:14:55,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-13T00:14:55,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731456895881, max timestamp=1731456895891Num uniques 1054; 2024-11-13T00:14:55,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-13T00:14:55,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456955924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:55,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731456895916, max timestamp=1731456895926Num uniques 1113; 2024-11-13T00:14:55,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731456895916, max timestamp=1731456895926Num uniques 1113; 2024-11-13T00:14:55,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731456895916, max timestamp=1731456895926Num uniques 1113; 2024-11-13T00:14:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-13T00:14:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-13T00:14:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-13T00:14:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:55,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731456895948, max timestamp=1731456895966Num uniques 1122; 2024-11-13T00:14:55,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-13T00:14:55,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731456895948, max timestamp=1731456895966Num uniques 1122; 2024-11-13T00:14:55,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-13T00:14:55,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731456895948, max timestamp=1731456895966Num uniques 1072; 2024-11-13T00:14:55,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-13T00:14:55,994 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8a279c4ce3304943856ad8515e4dd4bb 2024-11-13T00:14:55,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/2ecb996dd0ad44838c2549e0aa50de19 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/2ecb996dd0ad44838c2549e0aa50de19 2024-11-13T00:14:56,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731456895984, max timestamp=1731456896000Num uniques 1031; 2024-11-13T00:14:56,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-13T00:14:56,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731456895984, max timestamp=1731456896000Num uniques 1031; 2024-11-13T00:14:56,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-13T00:14:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731456895984, max timestamp=1731456896000Num uniques 1031; 2024-11-13T00:14:56,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=6870, suffix heap size=135192,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=150 2024-11-13T00:14:56,011 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/2ecb996dd0ad44838c2549e0aa50de19, entries=100, sequenceid=437, filesize=9.6 K 2024-11-13T00:14:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/23ef91b521134e40b37a5188bcc1343e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/23ef91b521134e40b37a5188bcc1343e 2024-11-13T00:14:56,016 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/23ef91b521134e40b37a5188bcc1343e, entries=100, sequenceid=437, filesize=9.6 K 2024-11-13T00:14:56,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8a279c4ce3304943856ad8515e4dd4bb as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a279c4ce3304943856ad8515e4dd4bb 2024-11-13T00:14:56,022 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a279c4ce3304943856ad8515e4dd4bb, entries=100, sequenceid=437, filesize=9.6 K 2024-11-13T00:14:56,022 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=83.60 KB/85604 for dc9547c78a88f5a8e277773a4b75b850 in 885ms, sequenceid=437, compaction requested=true 2024-11-13T00:14:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-13T00:14:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-13T00:14:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:56,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=90.31 KB heapSize=250.48 KB 2024-11-13T00:14:56,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:56,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:56,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:56,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:56,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:56,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:14:56,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T00:14:56,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5740 sec 2024-11-13T00:14:56,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.5810 sec 2024-11-13T00:14:56,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/08eac383e276478bbe96aa5bd89a8c83 is 50, key is test_row_0/A:col10/1731456896017/Put/seqid=0 2024-11-13T00:14:56,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742188_1364 (size=14741) 2024-11-13T00:14:56,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456956055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456956062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456956063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456956064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456956176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456956176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456956176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456956176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456956386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456956386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456956386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456956386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/08eac383e276478bbe96aa5bd89a8c83 2024-11-13T00:14:56,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/7b4101feafab4b6eaab6b4b4d0e6b383 is 50, key is test_row_0/B:col10/1731456896017/Put/seqid=0 2024-11-13T00:14:56,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742189_1365 (size=12301) 2024-11-13T00:14:56,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456956695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456956696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456956696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456956696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:56,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/7b4101feafab4b6eaab6b4b4d0e6b383 2024-11-13T00:14:56,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/953f13b0a40b488eb4ef93477d325b9e is 50, key is test_row_0/C:col10/1731456896017/Put/seqid=0 2024-11-13T00:14:56,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742190_1366 (size=12301) 2024-11-13T00:14:56,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.69 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/953f13b0a40b488eb4ef93477d325b9e 2024-11-13T00:14:56,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/08eac383e276478bbe96aa5bd89a8c83 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/08eac383e276478bbe96aa5bd89a8c83 2024-11-13T00:14:56,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/08eac383e276478bbe96aa5bd89a8c83, entries=200, sequenceid=604, filesize=14.4 K 2024-11-13T00:14:56,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/7b4101feafab4b6eaab6b4b4d0e6b383 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/7b4101feafab4b6eaab6b4b4d0e6b383 2024-11-13T00:14:56,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/7b4101feafab4b6eaab6b4b4d0e6b383, entries=150, sequenceid=604, filesize=12.0 K 2024-11-13T00:14:56,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/953f13b0a40b488eb4ef93477d325b9e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/953f13b0a40b488eb4ef93477d325b9e 2024-11-13T00:14:56,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/953f13b0a40b488eb4ef93477d325b9e, entries=150, sequenceid=604, filesize=12.0 K 2024-11-13T00:14:56,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~90.31 KB/92474, heapSize ~250.43 KB/256440, currentSize=93.93 KB/96180 for dc9547c78a88f5a8e277773a4b75b850 in 859ms, sequenceid=604, compaction requested=true 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:14:56,885 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:56,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:56,885 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:56,886 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:56,887 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:14:56,887 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:56,887 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a955924e719f4248885e812afe021592, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5cc9b9cfcd584eeca2eb419d684c8516, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/23ef91b521134e40b37a5188bcc1343e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/7b4101feafab4b6eaab6b4b4d0e6b383] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=46.4 K 2024-11-13T00:14:56,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52424 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:56,887 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a955924e719f4248885e812afe021592, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731456892585 2024-11-13T00:14:56,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:14:56,888 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:56,888 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a25abcd3461a4f08b91fb7e6d2f64f2e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/b74a948b9e2a48f2b65f8ab7e9f6c21d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/2ecb996dd0ad44838c2549e0aa50de19, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/08eac383e276478bbe96aa5bd89a8c83] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=51.2 K 2024-11-13T00:14:56,888 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cc9b9cfcd584eeca2eb419d684c8516, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731456893495 2024-11-13T00:14:56,888 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a25abcd3461a4f08b91fb7e6d2f64f2e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731456892585 2024-11-13T00:14:56,888 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ef91b521134e40b37a5188bcc1343e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731456893732 2024-11-13T00:14:56,888 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b74a948b9e2a48f2b65f8ab7e9f6c21d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731456893495 2024-11-13T00:14:56,889 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4101feafab4b6eaab6b4b4d0e6b383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731456896017 2024-11-13T00:14:56,889 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecb996dd0ad44838c2549e0aa50de19, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731456893732 2024-11-13T00:14:56,889 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08eac383e276478bbe96aa5bd89a8c83, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731456896016 2024-11-13T00:14:56,915 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:56,915 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/44ce577bb77d450283508d1cebf7abea is 50, key is test_row_0/B:col10/1731456896017/Put/seqid=0 2024-11-13T00:14:56,918 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:56,919 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/49e531d3851e4f5a8d06e1b1229e729f is 50, key is test_row_0/A:col10/1731456896017/Put/seqid=0 2024-11-13T00:14:56,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742191_1367 (size=13221) 2024-11-13T00:14:56,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742192_1368 (size=13221) 2024-11-13T00:14:56,941 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/44ce577bb77d450283508d1cebf7abea as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/44ce577bb77d450283508d1cebf7abea 2024-11-13T00:14:56,943 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/49e531d3851e4f5a8d06e1b1229e729f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/49e531d3851e4f5a8d06e1b1229e729f 2024-11-13T00:14:56,949 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into 44ce577bb77d450283508d1cebf7abea(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:56,949 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:56,949 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=12, startTime=1731456896885; duration=0sec 2024-11-13T00:14:56,949 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:14:56,949 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:14:56,950 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:14:56,950 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 49e531d3851e4f5a8d06e1b1229e729f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:56,950 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:56,950 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=12, startTime=1731456896885; duration=0sec 2024-11-13T00:14:56,950 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:56,950 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:14:56,951 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:14:56,951 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:14:56,951 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:56,951 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/24366c3d2b0942dc93fba4216623dfb6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/73427ef4558a4824a5198254de152b67, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a279c4ce3304943856ad8515e4dd4bb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/953f13b0a40b488eb4ef93477d325b9e] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=46.4 K 2024-11-13T00:14:56,952 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 24366c3d2b0942dc93fba4216623dfb6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731456892585 2024-11-13T00:14:56,953 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 73427ef4558a4824a5198254de152b67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731456893495 2024-11-13T00:14:56,953 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a279c4ce3304943856ad8515e4dd4bb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731456893732 2024-11-13T00:14:56,953 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 953f13b0a40b488eb4ef93477d325b9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731456896017 2024-11-13T00:14:56,967 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:14:56,968 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/c3f2eb84de5f408c9fdac46b04fe1c30 is 50, key is test_row_0/C:col10/1731456896017/Put/seqid=0 2024-11-13T00:14:56,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742193_1369 (size=13221) 2024-11-13T00:14:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:57,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T00:14:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:57,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/e993bd2b2e6343168fcdfe7c0bc99c3e is 50, key is test_row_0/A:col10/1731456897207/Put/seqid=0 2024-11-13T00:14:57,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742194_1370 (size=14741) 2024-11-13T00:14:57,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456957224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456957224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456957224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456957225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456957336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456957337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456957337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456957337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,385 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/c3f2eb84de5f408c9fdac46b04fe1c30 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/c3f2eb84de5f408c9fdac46b04fe1c30 2024-11-13T00:14:57,392 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into c3f2eb84de5f408c9fdac46b04fe1c30(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:14:57,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:57,392 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=12, startTime=1731456896885; duration=0sec 2024-11-13T00:14:57,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:14:57,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:14:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456957546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456957546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456957546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456957546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T00:14:57,606 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:57,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:14:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-13T00:14:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T00:14:57,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:14:57,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:14:57,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:14:57,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/e993bd2b2e6343168fcdfe7c0bc99c3e 2024-11-13T00:14:57,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/91b3b02bc7204092a8f937b528b007ef is 50, key is test_row_0/B:col10/1731456897207/Put/seqid=0 2024-11-13T00:14:57,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742195_1371 (size=12301) 2024-11-13T00:14:57,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/91b3b02bc7204092a8f937b528b007ef 2024-11-13T00:14:57,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b25475c630c4403bb74e4b59df7a7cfa is 50, key is test_row_0/C:col10/1731456897207/Put/seqid=0 2024-11-13T00:14:57,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742196_1372 (size=12301) 2024-11-13T00:14:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T00:14:57,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T00:14:57,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:57,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:57,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:57,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:57,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:57,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456957856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456957856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456957856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456957856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:57,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T00:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:57,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T00:14:58,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b25475c630c4403bb74e4b59df7a7cfa 2024-11-13T00:14:58,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/e993bd2b2e6343168fcdfe7c0bc99c3e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e993bd2b2e6343168fcdfe7c0bc99c3e 2024-11-13T00:14:58,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e993bd2b2e6343168fcdfe7c0bc99c3e, entries=200, sequenceid=625, filesize=14.4 K 2024-11-13T00:14:58,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/91b3b02bc7204092a8f937b528b007ef as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/91b3b02bc7204092a8f937b528b007ef 2024-11-13T00:14:58,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T00:14:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:58,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:14:58,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/91b3b02bc7204092a8f937b528b007ef, entries=150, sequenceid=625, filesize=12.0 K 2024-11-13T00:14:58,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/b25475c630c4403bb74e4b59df7a7cfa as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b25475c630c4403bb74e4b59df7a7cfa 2024-11-13T00:14:58,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b25475c630c4403bb74e4b59df7a7cfa, entries=150, sequenceid=625, filesize=12.0 K 2024-11-13T00:14:58,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for dc9547c78a88f5a8e277773a4b75b850 in 868ms, sequenceid=625, compaction requested=false 2024-11-13T00:14:58,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:58,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T00:14:58,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:58,222 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:58,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:58,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9801dcf2ce024166b1f9ec50285cdd09 is 50, key is test_row_0/A:col10/1731456897224/Put/seqid=0 2024-11-13T00:14:58,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742197_1373 (size=12301) 2024-11-13T00:14:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T00:14:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:58,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. as already flushing 2024-11-13T00:14:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456958388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456958389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456958390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456958390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,401 INFO [Thread-1310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1310. 2024-11-13T00:14:58,401 INFO [Thread-1314 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1314. 2024-11-13T00:14:58,401 DEBUG [Thread-1310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:58,401 DEBUG [Thread-1314 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:58,401 DEBUG [Thread-1310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,401 DEBUG [Thread-1314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,401 DEBUG [Thread-1310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:58,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:58,401 DEBUG [Thread-1314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,403 INFO [Thread-1316 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1316. 2024-11-13T00:14:58,403 DEBUG [Thread-1316 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:58,403 DEBUG [Thread-1316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,403 DEBUG [Thread-1316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:58,405 INFO [Thread-1318 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1318. 2024-11-13T00:14:58,405 DEBUG [Thread-1318 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:58,405 DEBUG [Thread-1318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,405 DEBUG [Thread-1318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:58,405 INFO [Thread-1312 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1312. 2024-11-13T00:14:58,405 DEBUG [Thread-1312 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:58,405 DEBUG [Thread-1312 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,405 DEBUG [Thread-1312 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:58,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456958496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456958506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456958506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456958506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,631 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9801dcf2ce024166b1f9ec50285cdd09 2024-11-13T00:14:58,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/de8a8715e4bb4b5d86e93239d2f4cabe is 50, key is test_row_0/B:col10/1731456897224/Put/seqid=0 2024-11-13T00:14:58,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742198_1374 (size=12301) 2024-11-13T00:14:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456958706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456958716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456958716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456958716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T00:14:59,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41754 deadline: 1731456959015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:59,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:59,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:59,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:59,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:14:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41828 deadline: 1731456959026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41848 deadline: 1731456959026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41856 deadline: 1731456959026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:14:59,041 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/de8a8715e4bb4b5d86e93239d2f4cabe 2024-11-13T00:14:59,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/2ad6f38d80554d0e96759a58d8ae7aac is 50, key is test_row_0/C:col10/1731456897224/Put/seqid=0 2024-11-13T00:14:59,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742199_1375 (size=12301) 2024-11-13T00:14:59,451 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=639 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/2ad6f38d80554d0e96759a58d8ae7aac 2024-11-13T00:14:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9801dcf2ce024166b1f9ec50285cdd09 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9801dcf2ce024166b1f9ec50285cdd09 2024-11-13T00:14:59,458 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9801dcf2ce024166b1f9ec50285cdd09, entries=150, sequenceid=639, filesize=12.0 K 2024-11-13T00:14:59,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/de8a8715e4bb4b5d86e93239d2f4cabe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/de8a8715e4bb4b5d86e93239d2f4cabe 2024-11-13T00:14:59,462 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/de8a8715e4bb4b5d86e93239d2f4cabe, entries=150, sequenceid=639, filesize=12.0 K 2024-11-13T00:14:59,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/2ad6f38d80554d0e96759a58d8ae7aac as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/2ad6f38d80554d0e96759a58d8ae7aac 2024-11-13T00:14:59,466 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/2ad6f38d80554d0e96759a58d8ae7aac, entries=150, sequenceid=639, filesize=12.0 K 2024-11-13T00:14:59,467 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for dc9547c78a88f5a8e277773a4b75b850 in 1244ms, sequenceid=639, compaction requested=true 2024-11-13T00:14:59,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:14:59,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:14:59,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-13T00:14:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-13T00:14:59,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T00:14:59,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-11-13T00:14:59,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.8630 sec 2024-11-13T00:14:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:14:59,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:14:59,528 INFO [Thread-1299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1299. 2024-11-13T00:14:59,528 DEBUG [Thread-1299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:59,528 DEBUG [Thread-1299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:14:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:59,528 DEBUG [Thread-1299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:14:59,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:14:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:14:59,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a5529c2e37cd47bca1352bce8113fc47 is 50, key is test_row_0/A:col10/1731456899526/Put/seqid=0 2024-11-13T00:14:59,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742200_1376 (size=12301) 2024-11-13T00:14:59,537 INFO [Thread-1301 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1301. 2024-11-13T00:14:59,537 INFO [Thread-1305 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1305. 2024-11-13T00:14:59,537 DEBUG [Thread-1301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:59,537 DEBUG [Thread-1301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,537 DEBUG [Thread-1305 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:59,537 DEBUG [Thread-1305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,537 INFO [Thread-1307 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1307. 2024-11-13T00:14:59,537 DEBUG [Thread-1301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,537 DEBUG [Thread-1307 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:14:59,537 DEBUG [Thread-1305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,537 DEBUG [Thread-1307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:59,537 DEBUG [Thread-1307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:14:59,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:59,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:14:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T00:14:59,757 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:14:59,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a5529c2e37cd47bca1352bce8113fc47 2024-11-13T00:14:59,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/f81a9962dbdf4b75a79491c0fd203f60 is 50, key is test_row_0/B:col10/1731456899526/Put/seqid=0 2024-11-13T00:14:59,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742201_1377 (size=12301) 2024-11-13T00:14:59,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/f81a9962dbdf4b75a79491c0fd203f60 2024-11-13T00:14:59,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/3c5009fe1add4a6bb2fc2273fdff9f8c is 50, key is test_row_0/C:col10/1731456899526/Put/seqid=0 2024-11-13T00:14:59,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742202_1378 (size=12301) 2024-11-13T00:15:00,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/3c5009fe1add4a6bb2fc2273fdff9f8c 2024-11-13T00:15:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/a5529c2e37cd47bca1352bce8113fc47 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a5529c2e37cd47bca1352bce8113fc47 2024-11-13T00:15:00,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a5529c2e37cd47bca1352bce8113fc47, entries=150, sequenceid=659, filesize=12.0 K 2024-11-13T00:15:00,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/f81a9962dbdf4b75a79491c0fd203f60 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/f81a9962dbdf4b75a79491c0fd203f60 2024-11-13T00:15:00,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/f81a9962dbdf4b75a79491c0fd203f60, entries=150, sequenceid=659, filesize=12.0 K 2024-11-13T00:15:00,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/3c5009fe1add4a6bb2fc2273fdff9f8c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/3c5009fe1add4a6bb2fc2273fdff9f8c 2024-11-13T00:15:00,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/3c5009fe1add4a6bb2fc2273fdff9f8c, entries=150, sequenceid=659, filesize=12.0 K 2024-11-13T00:15:00,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for dc9547c78a88f5a8e277773a4b75b850 in 862ms, sequenceid=659, compaction requested=true 2024-11-13T00:15:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:15:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:00,389 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:00,389 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc9547c78a88f5a8e277773a4b75b850:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:00,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:00,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/A is initiating minor compaction (all files) 2024-11-13T00:15:00,391 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/A in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:00,391 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/49e531d3851e4f5a8d06e1b1229e729f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e993bd2b2e6343168fcdfe7c0bc99c3e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9801dcf2ce024166b1f9ec50285cdd09, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a5529c2e37cd47bca1352bce8113fc47] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=51.3 K 2024-11-13T00:15:00,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:00,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/B is initiating minor compaction (all files) 2024-11-13T00:15:00,392 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/B in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:00,392 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e531d3851e4f5a8d06e1b1229e729f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731456896017 2024-11-13T00:15:00,392 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/44ce577bb77d450283508d1cebf7abea, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/91b3b02bc7204092a8f937b528b007ef, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/de8a8715e4bb4b5d86e93239d2f4cabe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/f81a9962dbdf4b75a79491c0fd203f60] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=48.9 K 2024-11-13T00:15:00,392 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e993bd2b2e6343168fcdfe7c0bc99c3e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731456896060 2024-11-13T00:15:00,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ce577bb77d450283508d1cebf7abea, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731456896017 2024-11-13T00:15:00,392 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9801dcf2ce024166b1f9ec50285cdd09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1731456897222 2024-11-13T00:15:00,393 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5529c2e37cd47bca1352bce8113fc47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=659, earliestPutTs=1731456898388 2024-11-13T00:15:00,393 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 91b3b02bc7204092a8f937b528b007ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731456896060 2024-11-13T00:15:00,393 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting de8a8715e4bb4b5d86e93239d2f4cabe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1731456897222 2024-11-13T00:15:00,394 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f81a9962dbdf4b75a79491c0fd203f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=659, earliestPutTs=1731456898388 2024-11-13T00:15:00,406 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#A#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:00,407 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/3238cfcf6901462da0cd2fca28521b0d is 50, key is test_row_0/A:col10/1731456899526/Put/seqid=0 2024-11-13T00:15:00,411 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:00,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/3db2beeb307f409a8bdbfbb5bd520846 is 50, key is test_row_0/B:col10/1731456899526/Put/seqid=0 2024-11-13T00:15:00,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742203_1379 (size=13357) 2024-11-13T00:15:00,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742204_1380 (size=13357) 2024-11-13T00:15:00,820 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/3238cfcf6901462da0cd2fca28521b0d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/3238cfcf6901462da0cd2fca28521b0d 2024-11-13T00:15:00,825 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/3db2beeb307f409a8bdbfbb5bd520846 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/3db2beeb307f409a8bdbfbb5bd520846 2024-11-13T00:15:00,825 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/A of dc9547c78a88f5a8e277773a4b75b850 into 3238cfcf6901462da0cd2fca28521b0d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:00,825 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:15:00,825 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/A, priority=12, startTime=1731456900389; duration=0sec 2024-11-13T00:15:00,825 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:00,825 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:A 2024-11-13T00:15:00,826 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:00,826 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:00,826 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): dc9547c78a88f5a8e277773a4b75b850/C is initiating minor compaction (all files) 2024-11-13T00:15:00,827 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc9547c78a88f5a8e277773a4b75b850/C in TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:00,827 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/c3f2eb84de5f408c9fdac46b04fe1c30, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b25475c630c4403bb74e4b59df7a7cfa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/2ad6f38d80554d0e96759a58d8ae7aac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/3c5009fe1add4a6bb2fc2273fdff9f8c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp, totalSize=48.9 K 2024-11-13T00:15:00,827 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f2eb84de5f408c9fdac46b04fe1c30, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731456896017 2024-11-13T00:15:00,827 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25475c630c4403bb74e4b59df7a7cfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1731456896060 2024-11-13T00:15:00,828 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad6f38d80554d0e96759a58d8ae7aac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=639, earliestPutTs=1731456897222 2024-11-13T00:15:00,828 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5009fe1add4a6bb2fc2273fdff9f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=659, earliestPutTs=1731456898388 2024-11-13T00:15:00,829 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/B of dc9547c78a88f5a8e277773a4b75b850 into 3db2beeb307f409a8bdbfbb5bd520846(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:00,829 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:15:00,829 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/B, priority=12, startTime=1731456900389; duration=0sec 2024-11-13T00:15:00,830 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:00,830 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:B 2024-11-13T00:15:00,835 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc9547c78a88f5a8e277773a4b75b850#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:00,836 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8d49b2861a30460895480637b7e5cfc2 is 50, key is test_row_0/C:col10/1731456899526/Put/seqid=0 2024-11-13T00:15:00,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742205_1381 (size=13357) 2024-11-13T00:15:01,078 INFO [Thread-1303 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1303. 2024-11-13T00:15:01,078 DEBUG [Thread-1303 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:01,078 DEBUG [Thread-1303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:01,078 DEBUG [Thread-1303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:01,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3249 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3220 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3323 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3239 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3228 2024-11-13T00:15:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T00:15:01,082 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:15:01,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:15:01,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:01,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:01,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:01,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T00:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T00:15:01,088 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456901088"}]},"ts":"1731456901088"} 2024-11-13T00:15:01,091 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T00:15:01,091 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T00:15:01,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:15:01,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, UNASSIGN}] 2024-11-13T00:15:01,095 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, UNASSIGN 2024-11-13T00:15:01,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=dc9547c78a88f5a8e277773a4b75b850, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:01,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, UNASSIGN because future has completed 2024-11-13T00:15:01,098 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:15:01,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T00:15:01,249 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/8d49b2861a30460895480637b7e5cfc2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8d49b2861a30460895480637b7e5cfc2 2024-11-13T00:15:01,251 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:15:01,251 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:15:01,251 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing dc9547c78a88f5a8e277773a4b75b850, disabling compactions & flushes 2024-11-13T00:15:01,251 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:01,256 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc9547c78a88f5a8e277773a4b75b850/C of dc9547c78a88f5a8e277773a4b75b850 into 8d49b2861a30460895480637b7e5cfc2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:01,256 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. after waiting 0 ms 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:01,256 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing dc9547c78a88f5a8e277773a4b75b850 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=A 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=B 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc9547c78a88f5a8e277773a4b75b850, store=C 2024-11-13T00:15:01,256 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:01,257 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc9547c78a88f5a8e277773a4b75b850: 2024-11-13T00:15:01,257 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., storeName=dc9547c78a88f5a8e277773a4b75b850/C, priority=12, startTime=1731456900390; duration=0sec 2024-11-13T00:15:01,258 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:01,258 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc9547c78a88f5a8e277773a4b75b850:C 2024-11-13T00:15:01,261 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9fbe848df2fa4b7aa851c91805aad5ae is 50, key is test_row_1/A:col10/1731456901076/Put/seqid=0 2024-11-13T00:15:01,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742206_1382 (size=9857) 2024-11-13T00:15:01,274 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9fbe848df2fa4b7aa851c91805aad5ae 2024-11-13T00:15:01,289 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ab182379fd3d45029e8f8346bd3eae20 is 50, key is test_row_1/B:col10/1731456901076/Put/seqid=0 2024-11-13T00:15:01,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742207_1383 (size=9857) 2024-11-13T00:15:01,300 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ab182379fd3d45029e8f8346bd3eae20 2024-11-13T00:15:01,307 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/01873117881449edac282a4ce3c5b1ae is 50, key is test_row_1/C:col10/1731456901076/Put/seqid=0 2024-11-13T00:15:01,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742208_1384 (size=9857) 2024-11-13T00:15:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T00:15:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T00:15:01,718 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/01873117881449edac282a4ce3c5b1ae 2024-11-13T00:15:01,724 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/A/9fbe848df2fa4b7aa851c91805aad5ae as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9fbe848df2fa4b7aa851c91805aad5ae 2024-11-13T00:15:01,728 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9fbe848df2fa4b7aa851c91805aad5ae, entries=100, sequenceid=669, filesize=9.6 K 2024-11-13T00:15:01,729 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/B/ab182379fd3d45029e8f8346bd3eae20 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ab182379fd3d45029e8f8346bd3eae20 2024-11-13T00:15:01,733 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ab182379fd3d45029e8f8346bd3eae20, entries=100, sequenceid=669, filesize=9.6 K 2024-11-13T00:15:01,734 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/.tmp/C/01873117881449edac282a4ce3c5b1ae as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/01873117881449edac282a4ce3c5b1ae 2024-11-13T00:15:01,737 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/01873117881449edac282a4ce3c5b1ae, entries=100, sequenceid=669, filesize=9.6 K 2024-11-13T00:15:01,738 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for dc9547c78a88f5a8e277773a4b75b850 in 482ms, sequenceid=669, compaction requested=false 2024-11-13T00:15:01,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1fd73b1dec04e9ca106ba550a48b3a4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/086bb8168e9146c594ede89f7c4b43ee, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/80bb50bf285d410f976e88a0d783f62f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7678836679ed4eaf9f487dc5acf4e357, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/935bce0da3f6498b8750d48ca1cd2af2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/712e5a09341a4da78d9e2ab546bfa5a6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/797b01d96fb447b4ad540b2cbe4956fe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ced4f2979e9345c98511bbcb2a1a03d1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/84e98859c37d47a5ad2ab606381ab96b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/536e83fbec3b472781c6a2dd26f1540e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/5b6956a49c7e4f1f83730ef549b46ffe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f7e0b28ea4a443d5a3db3188edf938fd, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ac5ce6ee6eab43d48041ed6e1e935cff, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f4afe67772f248af96ef58821663806c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/58d42d63eec240d8af5adf3805968462, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e93cd77e9647477eb171dc9998ed6f84, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f67bb64a4f047369736109330a7e4a6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/0721fa8ef5fc436ba061165023d62348, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/744426c36acc43978280d0bf79e7cda0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/8629eac2952d4817b27c38baf56e1e65, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1faeefab7c040b7b9efcc7585877c6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7c77aae6bff5405ca801ce2fa174f131, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a25abcd3461a4f08b91fb7e6d2f64f2e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/b74a948b9e2a48f2b65f8ab7e9f6c21d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/2ecb996dd0ad44838c2549e0aa50de19, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/08eac383e276478bbe96aa5bd89a8c83, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/49e531d3851e4f5a8d06e1b1229e729f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e993bd2b2e6343168fcdfe7c0bc99c3e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9801dcf2ce024166b1f9ec50285cdd09, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a5529c2e37cd47bca1352bce8113fc47] to archive 2024-11-13T00:15:01,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:01,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1fd73b1dec04e9ca106ba550a48b3a4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1fd73b1dec04e9ca106ba550a48b3a4 2024-11-13T00:15:01,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/086bb8168e9146c594ede89f7c4b43ee to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/086bb8168e9146c594ede89f7c4b43ee 2024-11-13T00:15:01,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/80bb50bf285d410f976e88a0d783f62f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/80bb50bf285d410f976e88a0d783f62f 2024-11-13T00:15:01,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7678836679ed4eaf9f487dc5acf4e357 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7678836679ed4eaf9f487dc5acf4e357 2024-11-13T00:15:01,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/935bce0da3f6498b8750d48ca1cd2af2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/935bce0da3f6498b8750d48ca1cd2af2 2024-11-13T00:15:01,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/712e5a09341a4da78d9e2ab546bfa5a6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/712e5a09341a4da78d9e2ab546bfa5a6 2024-11-13T00:15:01,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/797b01d96fb447b4ad540b2cbe4956fe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/797b01d96fb447b4ad540b2cbe4956fe 2024-11-13T00:15:01,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ced4f2979e9345c98511bbcb2a1a03d1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ced4f2979e9345c98511bbcb2a1a03d1 2024-11-13T00:15:01,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f0f8eb0c3bd467cb66b7496b4d1c1a9 2024-11-13T00:15:01,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/84e98859c37d47a5ad2ab606381ab96b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/84e98859c37d47a5ad2ab606381ab96b 2024-11-13T00:15:01,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/536e83fbec3b472781c6a2dd26f1540e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/536e83fbec3b472781c6a2dd26f1540e 2024-11-13T00:15:01,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/5b6956a49c7e4f1f83730ef549b46ffe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/5b6956a49c7e4f1f83730ef549b46ffe 2024-11-13T00:15:01,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f7e0b28ea4a443d5a3db3188edf938fd to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f7e0b28ea4a443d5a3db3188edf938fd 2024-11-13T00:15:01,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ac5ce6ee6eab43d48041ed6e1e935cff to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/ac5ce6ee6eab43d48041ed6e1e935cff 2024-11-13T00:15:01,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f4afe67772f248af96ef58821663806c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/f4afe67772f248af96ef58821663806c 2024-11-13T00:15:01,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/58d42d63eec240d8af5adf3805968462 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/58d42d63eec240d8af5adf3805968462 2024-11-13T00:15:01,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e93cd77e9647477eb171dc9998ed6f84 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e93cd77e9647477eb171dc9998ed6f84 2024-11-13T00:15:01,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f67bb64a4f047369736109330a7e4a6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9f67bb64a4f047369736109330a7e4a6 2024-11-13T00:15:01,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/0721fa8ef5fc436ba061165023d62348 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/0721fa8ef5fc436ba061165023d62348 2024-11-13T00:15:01,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/744426c36acc43978280d0bf79e7cda0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/744426c36acc43978280d0bf79e7cda0 2024-11-13T00:15:01,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/8629eac2952d4817b27c38baf56e1e65 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/8629eac2952d4817b27c38baf56e1e65 2024-11-13T00:15:01,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1faeefab7c040b7b9efcc7585877c6d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a1faeefab7c040b7b9efcc7585877c6d 2024-11-13T00:15:01,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7c77aae6bff5405ca801ce2fa174f131 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/7c77aae6bff5405ca801ce2fa174f131 2024-11-13T00:15:01,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a25abcd3461a4f08b91fb7e6d2f64f2e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a25abcd3461a4f08b91fb7e6d2f64f2e 2024-11-13T00:15:01,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/b74a948b9e2a48f2b65f8ab7e9f6c21d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/b74a948b9e2a48f2b65f8ab7e9f6c21d 2024-11-13T00:15:01,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/2ecb996dd0ad44838c2549e0aa50de19 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/2ecb996dd0ad44838c2549e0aa50de19 2024-11-13T00:15:01,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/08eac383e276478bbe96aa5bd89a8c83 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/08eac383e276478bbe96aa5bd89a8c83 2024-11-13T00:15:01,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/49e531d3851e4f5a8d06e1b1229e729f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/49e531d3851e4f5a8d06e1b1229e729f 2024-11-13T00:15:01,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e993bd2b2e6343168fcdfe7c0bc99c3e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/e993bd2b2e6343168fcdfe7c0bc99c3e 2024-11-13T00:15:01,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9801dcf2ce024166b1f9ec50285cdd09 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9801dcf2ce024166b1f9ec50285cdd09 2024-11-13T00:15:01,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a5529c2e37cd47bca1352bce8113fc47 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/a5529c2e37cd47bca1352bce8113fc47 2024-11-13T00:15:01,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/316dc7bd98b34abda6ba9accce52e823, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a2b1c968b517469e8ca56747452d7464, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ff6d9a87d82e44aa93315883cb570e1a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5a5f8813be14bf590a03ceaacebbf3e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/685d3db5ed134d829dc79e4d4f5160ba, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/8834835a36ce45969c0e96451cb9e491, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b51feb39c6c94f4a90cfccb8a58fed1e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/435d1bde48ef41b0a25ba24b7494081c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b15f6f6ee1b94fd88b99e05ff45249a9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/0100d60289044138a0b90e262296bf8f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/50f082c571b4468285ed19c01c978fc1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5c2f034873b6477aba30c241be5e3752, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e727e20b20df4114ac9117a2c7322116, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/135cc02e0c98428fb6a06fe986d3a63c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5552657195a47d69b752b668f8625e0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a88786eac46400bbbeda7c4a544305c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/6ca758c54334443da13cbe9dec6f1f40, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a7fd697295143699c15a3f7fb076106, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5ccccd7d6db94e51ac51edd021fd3f2c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a479b084db644762b870894bd183a0b8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/cff6381f4873478da05abe6df9829b26, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ae65362331ea4fd899bc5f6af4d8ae06, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a955924e719f4248885e812afe021592, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/58e193c0b5bb4ac5b002f63a96ab3be3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5cc9b9cfcd584eeca2eb419d684c8516, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/23ef91b521134e40b37a5188bcc1343e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/44ce577bb77d450283508d1cebf7abea, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/7b4101feafab4b6eaab6b4b4d0e6b383, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/91b3b02bc7204092a8f937b528b007ef, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/de8a8715e4bb4b5d86e93239d2f4cabe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/f81a9962dbdf4b75a79491c0fd203f60] to archive 2024-11-13T00:15:01,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:01,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/316dc7bd98b34abda6ba9accce52e823 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/316dc7bd98b34abda6ba9accce52e823 2024-11-13T00:15:01,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a2b1c968b517469e8ca56747452d7464 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a2b1c968b517469e8ca56747452d7464 2024-11-13T00:15:01,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ff6d9a87d82e44aa93315883cb570e1a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ff6d9a87d82e44aa93315883cb570e1a 2024-11-13T00:15:01,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5a5f8813be14bf590a03ceaacebbf3e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5a5f8813be14bf590a03ceaacebbf3e 2024-11-13T00:15:01,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/685d3db5ed134d829dc79e4d4f5160ba to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/685d3db5ed134d829dc79e4d4f5160ba 2024-11-13T00:15:01,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/8834835a36ce45969c0e96451cb9e491 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/8834835a36ce45969c0e96451cb9e491 2024-11-13T00:15:01,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b51feb39c6c94f4a90cfccb8a58fed1e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b51feb39c6c94f4a90cfccb8a58fed1e 2024-11-13T00:15:01,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/435d1bde48ef41b0a25ba24b7494081c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/435d1bde48ef41b0a25ba24b7494081c 2024-11-13T00:15:01,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b15f6f6ee1b94fd88b99e05ff45249a9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/b15f6f6ee1b94fd88b99e05ff45249a9 2024-11-13T00:15:01,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/0100d60289044138a0b90e262296bf8f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/0100d60289044138a0b90e262296bf8f 2024-11-13T00:15:01,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/50f082c571b4468285ed19c01c978fc1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/50f082c571b4468285ed19c01c978fc1 2024-11-13T00:15:01,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5c2f034873b6477aba30c241be5e3752 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5c2f034873b6477aba30c241be5e3752 2024-11-13T00:15:01,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e727e20b20df4114ac9117a2c7322116 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e727e20b20df4114ac9117a2c7322116 2024-11-13T00:15:01,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/135cc02e0c98428fb6a06fe986d3a63c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/135cc02e0c98428fb6a06fe986d3a63c 2024-11-13T00:15:01,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5552657195a47d69b752b668f8625e0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/e5552657195a47d69b752b668f8625e0 2024-11-13T00:15:01,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a88786eac46400bbbeda7c4a544305c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a88786eac46400bbbeda7c4a544305c 2024-11-13T00:15:01,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/6ca758c54334443da13cbe9dec6f1f40 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/6ca758c54334443da13cbe9dec6f1f40 2024-11-13T00:15:01,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a7fd697295143699c15a3f7fb076106 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/1a7fd697295143699c15a3f7fb076106 2024-11-13T00:15:01,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5ccccd7d6db94e51ac51edd021fd3f2c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5ccccd7d6db94e51ac51edd021fd3f2c 2024-11-13T00:15:01,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a479b084db644762b870894bd183a0b8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a479b084db644762b870894bd183a0b8 2024-11-13T00:15:01,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/cff6381f4873478da05abe6df9829b26 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/cff6381f4873478da05abe6df9829b26 2024-11-13T00:15:01,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ae65362331ea4fd899bc5f6af4d8ae06 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ae65362331ea4fd899bc5f6af4d8ae06 2024-11-13T00:15:01,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a955924e719f4248885e812afe021592 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/a955924e719f4248885e812afe021592 2024-11-13T00:15:01,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/58e193c0b5bb4ac5b002f63a96ab3be3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/58e193c0b5bb4ac5b002f63a96ab3be3 2024-11-13T00:15:01,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5cc9b9cfcd584eeca2eb419d684c8516 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/5cc9b9cfcd584eeca2eb419d684c8516 2024-11-13T00:15:01,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/23ef91b521134e40b37a5188bcc1343e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/23ef91b521134e40b37a5188bcc1343e 2024-11-13T00:15:01,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/44ce577bb77d450283508d1cebf7abea to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/44ce577bb77d450283508d1cebf7abea 2024-11-13T00:15:01,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/7b4101feafab4b6eaab6b4b4d0e6b383 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/7b4101feafab4b6eaab6b4b4d0e6b383 2024-11-13T00:15:01,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/91b3b02bc7204092a8f937b528b007ef to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/91b3b02bc7204092a8f937b528b007ef 2024-11-13T00:15:01,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/de8a8715e4bb4b5d86e93239d2f4cabe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/de8a8715e4bb4b5d86e93239d2f4cabe 2024-11-13T00:15:01,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/f81a9962dbdf4b75a79491c0fd203f60 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/f81a9962dbdf4b75a79491c0fd203f60 2024-11-13T00:15:01,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/0e351485c6cf497fb8da825570ebadd6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e1e60dec84d341f5a5b783aa918e634e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/51375b48228f4f09a25e5f28d45b2050, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/d9a04bfe3a58436a988f1abda68f07c2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b2a3b74156964a01b61a58ac8d762ee9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/64ecb204bbef45ee8a1b1e12dddaf874, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e74945a9a9b84f5cb718a5d01f4f2b5b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/82a132bc78074c549632847852eea940, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/439fcdc23bbd41ba934477effe52a614, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a64546fb2304642a5f60430119b88da, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/52ee682ce4804609bb02348cf963beaa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/1331c85093794f24bfab7260c610460a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e622cb53789d489f95342b88825c0b1c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/93617abd70bf4fcd887e2144d37506ea, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9df0477ad9cd46ba88b66052ca69d844, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9e6a3f7e85694edbb0ac6fb4531d4d6d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/054f3295c6c04eddbc583d5f0ecc5b8c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/292ca81eacfb49efa0d7673ba5616ec7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9ca3ca14f0314f67a0a4d110741fd112, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/172c5929948d4bb4ba7628852f4191fb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/40d7f1b8350d46b69fe4ab274e3401a3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/24366c3d2b0942dc93fba4216623dfb6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/830a5d136420480380f5e4901f3c1a36, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/73427ef4558a4824a5198254de152b67, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a279c4ce3304943856ad8515e4dd4bb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/c3f2eb84de5f408c9fdac46b04fe1c30, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/953f13b0a40b488eb4ef93477d325b9e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b25475c630c4403bb74e4b59df7a7cfa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/2ad6f38d80554d0e96759a58d8ae7aac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/3c5009fe1add4a6bb2fc2273fdff9f8c] to archive 2024-11-13T00:15:01,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:01,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/0e351485c6cf497fb8da825570ebadd6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/0e351485c6cf497fb8da825570ebadd6 2024-11-13T00:15:01,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e1e60dec84d341f5a5b783aa918e634e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e1e60dec84d341f5a5b783aa918e634e 2024-11-13T00:15:01,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/51375b48228f4f09a25e5f28d45b2050 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/51375b48228f4f09a25e5f28d45b2050 2024-11-13T00:15:01,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/d9a04bfe3a58436a988f1abda68f07c2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/d9a04bfe3a58436a988f1abda68f07c2 2024-11-13T00:15:01,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b2a3b74156964a01b61a58ac8d762ee9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b2a3b74156964a01b61a58ac8d762ee9 2024-11-13T00:15:01,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/64ecb204bbef45ee8a1b1e12dddaf874 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/64ecb204bbef45ee8a1b1e12dddaf874 2024-11-13T00:15:01,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e74945a9a9b84f5cb718a5d01f4f2b5b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e74945a9a9b84f5cb718a5d01f4f2b5b 2024-11-13T00:15:01,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/82a132bc78074c549632847852eea940 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/82a132bc78074c549632847852eea940 2024-11-13T00:15:01,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b07e6431f0b04b7f9a3bc0d47f3e4c2b 2024-11-13T00:15:01,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/439fcdc23bbd41ba934477effe52a614 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/439fcdc23bbd41ba934477effe52a614 2024-11-13T00:15:01,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a64546fb2304642a5f60430119b88da to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a64546fb2304642a5f60430119b88da 2024-11-13T00:15:01,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/52ee682ce4804609bb02348cf963beaa to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/52ee682ce4804609bb02348cf963beaa 2024-11-13T00:15:01,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/1331c85093794f24bfab7260c610460a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/1331c85093794f24bfab7260c610460a 2024-11-13T00:15:01,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e622cb53789d489f95342b88825c0b1c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/e622cb53789d489f95342b88825c0b1c 2024-11-13T00:15:01,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/93617abd70bf4fcd887e2144d37506ea to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/93617abd70bf4fcd887e2144d37506ea 2024-11-13T00:15:01,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9df0477ad9cd46ba88b66052ca69d844 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9df0477ad9cd46ba88b66052ca69d844 2024-11-13T00:15:01,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9e6a3f7e85694edbb0ac6fb4531d4d6d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9e6a3f7e85694edbb0ac6fb4531d4d6d 2024-11-13T00:15:01,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/054f3295c6c04eddbc583d5f0ecc5b8c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/054f3295c6c04eddbc583d5f0ecc5b8c 2024-11-13T00:15:01,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/292ca81eacfb49efa0d7673ba5616ec7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/292ca81eacfb49efa0d7673ba5616ec7 2024-11-13T00:15:01,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9ca3ca14f0314f67a0a4d110741fd112 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/9ca3ca14f0314f67a0a4d110741fd112 2024-11-13T00:15:01,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/172c5929948d4bb4ba7628852f4191fb to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/172c5929948d4bb4ba7628852f4191fb 2024-11-13T00:15:01,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/40d7f1b8350d46b69fe4ab274e3401a3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/40d7f1b8350d46b69fe4ab274e3401a3 2024-11-13T00:15:01,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/24366c3d2b0942dc93fba4216623dfb6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/24366c3d2b0942dc93fba4216623dfb6 2024-11-13T00:15:01,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/830a5d136420480380f5e4901f3c1a36 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/830a5d136420480380f5e4901f3c1a36 2024-11-13T00:15:01,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/73427ef4558a4824a5198254de152b67 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/73427ef4558a4824a5198254de152b67 2024-11-13T00:15:01,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a279c4ce3304943856ad8515e4dd4bb to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8a279c4ce3304943856ad8515e4dd4bb 2024-11-13T00:15:01,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/c3f2eb84de5f408c9fdac46b04fe1c30 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/c3f2eb84de5f408c9fdac46b04fe1c30 2024-11-13T00:15:01,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/953f13b0a40b488eb4ef93477d325b9e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/953f13b0a40b488eb4ef93477d325b9e 2024-11-13T00:15:01,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b25475c630c4403bb74e4b59df7a7cfa to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/b25475c630c4403bb74e4b59df7a7cfa 2024-11-13T00:15:01,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/2ad6f38d80554d0e96759a58d8ae7aac to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/2ad6f38d80554d0e96759a58d8ae7aac 2024-11-13T00:15:01,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/3c5009fe1add4a6bb2fc2273fdff9f8c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/3c5009fe1add4a6bb2fc2273fdff9f8c 2024-11-13T00:15:01,896 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/recovered.edits/672.seqid, newMaxSeqId=672, maxSeqId=1 2024-11-13T00:15:01,897 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. 2024-11-13T00:15:01,897 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for dc9547c78a88f5a8e277773a4b75b850: Waiting for close lock at 1731456901251Running coprocessor pre-close hooks at 1731456901251Disabling compacts and flushes for region at 1731456901251Disabling writes for close at 1731456901256 (+5 ms)Obtaining lock to block concurrent updates at 1731456901256Preparing flush snapshotting stores in dc9547c78a88f5a8e277773a4b75b850 at 1731456901256Finished memstore snapshotting TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731456901257 (+1 ms)Flushing stores of TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850. at 1731456901257Flushing dc9547c78a88f5a8e277773a4b75b850/A: creating writer at 1731456901257Flushing dc9547c78a88f5a8e277773a4b75b850/A: appending metadata at 1731456901260 (+3 ms)Flushing dc9547c78a88f5a8e277773a4b75b850/A: closing flushed file at 1731456901260Flushing dc9547c78a88f5a8e277773a4b75b850/B: creating writer at 1731456901283 (+23 ms)Flushing dc9547c78a88f5a8e277773a4b75b850/B: appending metadata at 1731456901288 (+5 ms)Flushing dc9547c78a88f5a8e277773a4b75b850/B: closing flushed file at 1731456901288Flushing dc9547c78a88f5a8e277773a4b75b850/C: creating writer at 1731456901304 (+16 ms)Flushing dc9547c78a88f5a8e277773a4b75b850/C: appending metadata at 1731456901306 (+2 ms)Flushing dc9547c78a88f5a8e277773a4b75b850/C: closing flushed file at 1731456901306Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33311cb9: reopening flushed file at 1731456901723 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@125fc375: reopening flushed file at 1731456901729 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72479b59: reopening flushed file at 1731456901733 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for dc9547c78a88f5a8e277773a4b75b850 in 482ms, sequenceid=669, compaction requested=false at 1731456901738 (+5 ms)Writing region close event to WAL at 1731456901892 (+154 ms)Running coprocessor post-close hooks at 1731456901896 (+4 ms)Closed at 1731456901896 2024-11-13T00:15:01,899 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:15:01,900 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=dc9547c78a88f5a8e277773a4b75b850, regionState=CLOSED 2024-11-13T00:15:01,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:01,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T00:15:01,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure dc9547c78a88f5a8e277773a4b75b850, server=cb28580903fb,45273,1731456804594 in 805 msec 2024-11-13T00:15:01,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T00:15:01,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc9547c78a88f5a8e277773a4b75b850, UNASSIGN in 812 msec 2024-11-13T00:15:01,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-13T00:15:01,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 818 msec 2024-11-13T00:15:01,914 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456901913"}]},"ts":"1731456901913"} 2024-11-13T00:15:01,915 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T00:15:01,915 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T00:15:01,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 832 msec 2024-11-13T00:15:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T00:15:02,226 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:02,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T00:15:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,228 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-13T00:15:02,229 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,231 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:15:02,233 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/recovered.edits] 2024-11-13T00:15:02,235 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/3238cfcf6901462da0cd2fca28521b0d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/3238cfcf6901462da0cd2fca28521b0d 2024-11-13T00:15:02,236 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9fbe848df2fa4b7aa851c91805aad5ae to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/A/9fbe848df2fa4b7aa851c91805aad5ae 2024-11-13T00:15:02,238 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/3db2beeb307f409a8bdbfbb5bd520846 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/3db2beeb307f409a8bdbfbb5bd520846 2024-11-13T00:15:02,239 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ab182379fd3d45029e8f8346bd3eae20 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/B/ab182379fd3d45029e8f8346bd3eae20 2024-11-13T00:15:02,241 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/01873117881449edac282a4ce3c5b1ae to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/01873117881449edac282a4ce3c5b1ae 2024-11-13T00:15:02,242 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8d49b2861a30460895480637b7e5cfc2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/C/8d49b2861a30460895480637b7e5cfc2 2024-11-13T00:15:02,245 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/recovered.edits/672.seqid to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850/recovered.edits/672.seqid 2024-11-13T00:15:02,245 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/dc9547c78a88f5a8e277773a4b75b850 2024-11-13T00:15:02,245 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T00:15:02,247 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,249 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T00:15:02,251 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T00:15:02,252 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,252 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T00:15:02,252 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731456902252"}]},"ts":"9223372036854775807"} 2024-11-13T00:15:02,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T00:15:02,254 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dc9547c78a88f5a8e277773a4b75b850, NAME => 'TestAcidGuarantees,,1731456876202.dc9547c78a88f5a8e277773a4b75b850.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T00:15:02,254 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T00:15:02,254 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731456902254"}]},"ts":"9223372036854775807"} 2024-11-13T00:15:02,256 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T00:15:02,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-13T00:15:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-13T00:15:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T00:15:02,336 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:02,346 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1970940896_22 at /127.0.0.1:55906 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_879911344_22 at /127.0.0.1:55890 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_879911344_22 at /127.0.0.1:36448 [Waiting for operation #230] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1970940896_22 at /127.0.0.1:48542 [Waiting for operation #364] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=442 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=933 (was 1561) 2024-11-13T00:15:02,355 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=442, ProcessCount=11, AvailableMemoryMB=932 2024-11-13T00:15:02,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:15:02,357 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:15:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:02,358 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:15:02,359 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:02,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-13T00:15:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T00:15:02,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:15:02,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742209_1385 (size=1061) 2024-11-13T00:15:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T00:15:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T00:15:02,767 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:15:02,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742210_1386 (size=53) 2024-11-13T00:15:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T00:15:03,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:15:03,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 96d540c45076fbf64e35c80767cd2e16, disabling compactions & flushes 2024-11-13T00:15:03,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. after waiting 0 ms 2024-11-13T00:15:03,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96d540c45076fbf64e35c80767cd2e16: Waiting for close lock at 1731456903172Disabling compacts and flushes for region at 1731456903172Disabling writes for close at 1731456903172Writing region close event to WAL at 1731456903172Closed at 1731456903172 2024-11-13T00:15:03,173 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:15:03,174 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731456903173"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731456903173"}]},"ts":"1731456903173"} 2024-11-13T00:15:03,175 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:15:03,176 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:15:03,176 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456903176"}]},"ts":"1731456903176"} 2024-11-13T00:15:03,177 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T00:15:03,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, ASSIGN}] 2024-11-13T00:15:03,178 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, ASSIGN 2024-11-13T00:15:03,178 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, ASSIGN; state=OFFLINE, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=false 2024-11-13T00:15:03,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:03,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, ASSIGN because future has completed 2024-11-13T00:15:03,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:03,485 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,485 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:15:03,486 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,486 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:15:03,486 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,486 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,487 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,488 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:03,488 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96d540c45076fbf64e35c80767cd2e16 columnFamilyName A 2024-11-13T00:15:03,489 DEBUG [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:03,489 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(327): Store=96d540c45076fbf64e35c80767cd2e16/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:03,489 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,490 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:03,490 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96d540c45076fbf64e35c80767cd2e16 columnFamilyName B 2024-11-13T00:15:03,490 DEBUG [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:03,491 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(327): Store=96d540c45076fbf64e35c80767cd2e16/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:03,491 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,492 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:03,492 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96d540c45076fbf64e35c80767cd2e16 columnFamilyName C 2024-11-13T00:15:03,492 DEBUG [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:03,492 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(327): Store=96d540c45076fbf64e35c80767cd2e16/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:03,492 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,492 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,493 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,493 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,494 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,494 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,494 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:15:03,495 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T00:15:03,496 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:15:03,497 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 96d540c45076fbf64e35c80767cd2e16; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74035739, jitterRate=0.10321848094463348}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:15:03,497 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:03,497 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 96d540c45076fbf64e35c80767cd2e16: Running coprocessor pre-open hook at 1731456903486Writing region info on filesystem at 1731456903486Initializing all the Stores at 1731456903486Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456903487 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456903487Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456903487Cleaning up temporary data from old regions at 1731456903494 (+7 ms)Running coprocessor post-open hooks at 1731456903497 (+3 ms)Region opened successfully at 1731456903497 2024-11-13T00:15:03,498 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., pid=93, masterSystemTime=1731456903482 2024-11-13T00:15:03,499 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,499 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:03,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:03,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:03,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-13T00:15:03,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 in 171 msec 2024-11-13T00:15:03,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T00:15:03,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, ASSIGN in 326 msec 2024-11-13T00:15:03,505 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:15:03,505 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456903505"}]},"ts":"1731456903505"} 2024-11-13T00:15:03,506 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T00:15:03,507 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:15:03,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-13T00:15:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T00:15:04,506 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:04,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:04,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:04,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:04,509 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:04,510 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa1b1f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:04,511 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:04,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:04,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:15:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:15:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ed7d70d 2024-11-13T00:15:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:15:04,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:15:04,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:15:04,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:15:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:04,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742211_1387 (size=1097) 2024-11-13T00:15:04,928 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T00:15:04,928 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T00:15:04,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:15:04,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, REOPEN/MOVE}] 2024-11-13T00:15:04,933 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, REOPEN/MOVE 2024-11-13T00:15:04,933 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:04,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, REOPEN/MOVE because future has completed 2024-11-13T00:15:04,934 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:15:04,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:05,087 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,087 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:15:05,087 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 96d540c45076fbf64e35c80767cd2e16, disabling compactions & flushes 2024-11-13T00:15:05,087 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,087 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,087 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. after waiting 0 ms 2024-11-13T00:15:05,087 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,090 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T00:15:05,091 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,091 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 96d540c45076fbf64e35c80767cd2e16: Waiting for close lock at 1731456905087Running coprocessor pre-close hooks at 1731456905087Disabling compacts and flushes for region at 1731456905087Disabling writes for close at 1731456905087Writing region close event to WAL at 1731456905087Running coprocessor post-close hooks at 1731456905091 (+4 ms)Closed at 1731456905091 2024-11-13T00:15:05,091 WARN [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 96d540c45076fbf64e35c80767cd2e16 to self. 2024-11-13T00:15:05,092 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,093 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=CLOSED 2024-11-13T00:15:05,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:05,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-13T00:15:05,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 in 160 msec 2024-11-13T00:15:05,096 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, REOPEN/MOVE; state=CLOSED, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=true 2024-11-13T00:15:05,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, REOPEN/MOVE because future has completed 2024-11-13T00:15:05,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:05,403 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,403 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:15:05,404 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,404 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:15:05,404 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,404 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,405 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,406 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:05,406 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96d540c45076fbf64e35c80767cd2e16 columnFamilyName A 2024-11-13T00:15:05,407 DEBUG [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:05,407 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(327): Store=96d540c45076fbf64e35c80767cd2e16/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:05,407 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,408 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:05,408 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96d540c45076fbf64e35c80767cd2e16 columnFamilyName B 2024-11-13T00:15:05,408 DEBUG [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:05,409 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(327): Store=96d540c45076fbf64e35c80767cd2e16/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:05,409 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,409 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:05,409 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96d540c45076fbf64e35c80767cd2e16 columnFamilyName C 2024-11-13T00:15:05,409 DEBUG [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:05,409 INFO [StoreOpener-96d540c45076fbf64e35c80767cd2e16-1 {}] regionserver.HStore(327): Store=96d540c45076fbf64e35c80767cd2e16/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:05,410 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,410 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,410 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,411 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,412 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,412 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,416 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:15:05,418 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,418 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 96d540c45076fbf64e35c80767cd2e16; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65189085, jitterRate=-0.02860693633556366}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:15:05,418 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,419 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 96d540c45076fbf64e35c80767cd2e16: Running coprocessor pre-open hook at 1731456905404Writing region info on filesystem at 1731456905404Initializing all the Stores at 1731456905404Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456905405 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456905405Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456905405Cleaning up temporary data from old regions at 1731456905412 (+7 ms)Running coprocessor post-open hooks at 1731456905418 (+6 ms)Region opened successfully at 1731456905419 (+1 ms) 2024-11-13T00:15:05,420 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., pid=98, masterSystemTime=1731456905400 2024-11-13T00:15:05,421 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,421 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,422 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=OPEN, openSeqNum=5, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:05,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-13T00:15:05,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 in 175 msec 2024-11-13T00:15:05,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T00:15:05,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, REOPEN/MOVE in 493 msec 2024-11-13T00:15:05,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-13T00:15:05,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-13T00:15:05,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-11-13T00:15:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-13T00:15:05,431 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:05,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c094db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,434 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f466625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1051ef11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19fb4331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a265ff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6927384f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,441 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb47e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,443 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d78b45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,444 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1327f4dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,446 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ef3a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,447 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc0f005, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,448 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622ee11d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,451 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5666c6ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,453 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e68d39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7417bf05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b3fad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,458 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60cd96e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:05,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:05,459 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3c078e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:05,461 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ac065d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:05,462 DEBUG [Thread-1745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,462 DEBUG [Thread-1756 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,462 DEBUG [Thread-1748 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,462 DEBUG [Thread-1750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,462 DEBUG [Thread-1737 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,462 DEBUG [Thread-1754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,463 DEBUG [Thread-1752 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,463 DEBUG [Thread-1756 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,463 DEBUG [Thread-1756 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,463 DEBUG [Thread-1743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:05,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,466 DEBUG [Thread-1739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T00:15:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T00:15:05,468 DEBUG [Thread-1741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:05,468 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:05,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:15:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:05,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138b0554a842f7416bb7f87a9758ed2316_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456905470/Put/seqid=0 2024-11-13T00:15:05,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742212_1388 (size=14594) 2024-11-13T00:15:05,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456905504, max timestamp=1731456905520Num uniques 859; 2024-11-13T00:15:05,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:05,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456905504, max timestamp=1731456905520Num uniques 859; 2024-11-13T00:15:05,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:05,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456905504, max timestamp=1731456905520Num uniques 859; 2024-11-13T00:15:05,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456965550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456965551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456965552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456965554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456965561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T00:15:05,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T00:15:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456965666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456965666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456965676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456965676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456965676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T00:15:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T00:15:05,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456965886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456965886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456965896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456965896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456965896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:05,920 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:05,923 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138b0554a842f7416bb7f87a9758ed2316_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138b0554a842f7416bb7f87a9758ed2316_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:05,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6df43ce7d604491096520075d316eb82, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:05,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6df43ce7d604491096520075d316eb82 is 175, key is test_row_0/A:col10/1731456905470/Put/seqid=0 2024-11-13T00:15:05,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T00:15:05,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:05,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742213_1389 (size=39549) 2024-11-13T00:15:05,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,929 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6df43ce7d604491096520075d316eb82 2024-11-13T00:15:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:05,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/34307627197e4bd5b83aea31b0a11726 is 50, key is test_row_0/B:col10/1731456905470/Put/seqid=0 2024-11-13T00:15:05,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742214_1390 (size=12001) 2024-11-13T00:15:05,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/34307627197e4bd5b83aea31b0a11726 2024-11-13T00:15:05,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/a7cadaee62174aea9405501781eaff0b is 50, key is test_row_0/C:col10/1731456905470/Put/seqid=0 2024-11-13T00:15:05,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742215_1391 (size=12001) 2024-11-13T00:15:05,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/a7cadaee62174aea9405501781eaff0b 2024-11-13T00:15:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6df43ce7d604491096520075d316eb82 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82 2024-11-13T00:15:05,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82, entries=200, sequenceid=14, filesize=38.6 K 2024-11-13T00:15:05,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/34307627197e4bd5b83aea31b0a11726 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/34307627197e4bd5b83aea31b0a11726 2024-11-13T00:15:05,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/34307627197e4bd5b83aea31b0a11726, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T00:15:05,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/a7cadaee62174aea9405501781eaff0b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a7cadaee62174aea9405501781eaff0b 2024-11-13T00:15:05,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a7cadaee62174aea9405501781eaff0b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T00:15:05,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 96d540c45076fbf64e35c80767cd2e16 in 518ms, sequenceid=14, compaction requested=false 2024-11-13T00:15:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T00:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:06,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:06,082 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b53a3589b23b47ce80382b5048ab1834_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:06,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742216_1392 (size=12154) 2024-11-13T00:15:06,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T00:15:06,096 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b53a3589b23b47ce80382b5048ab1834_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b53a3589b23b47ce80382b5048ab1834_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9b6eb547d24249848c1f9cce2a45395f, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9b6eb547d24249848c1f9cce2a45395f is 175, key is test_row_0/A:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:06,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742217_1393 (size=30955) 2024-11-13T00:15:06,103 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9b6eb547d24249848c1f9cce2a45395f 2024-11-13T00:15:06,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/4e6568e0bcde4263a481c7e6ad5e5342 is 50, key is test_row_0/B:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:06,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742218_1394 (size=12001) 2024-11-13T00:15:06,118 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/4e6568e0bcde4263a481c7e6ad5e5342 2024-11-13T00:15:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/17d1d2a377a9417bb4edd1708e2fdb42 is 50, key is test_row_0/C:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:06,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742219_1395 (size=12001) 2024-11-13T00:15:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:06,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456966218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456966218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456966230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456966231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456966231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456966336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456966336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456966336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456966355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456966356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,531 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/17d1d2a377a9417bb4edd1708e2fdb42 2024-11-13T00:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9b6eb547d24249848c1f9cce2a45395f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f 2024-11-13T00:15:06,540 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f, entries=150, sequenceid=49, filesize=30.2 K 2024-11-13T00:15:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/4e6568e0bcde4263a481c7e6ad5e5342 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4e6568e0bcde4263a481c7e6ad5e5342 2024-11-13T00:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,545 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4e6568e0bcde4263a481c7e6ad5e5342, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T00:15:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/17d1d2a377a9417bb4edd1708e2fdb42 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17d1d2a377a9417bb4edd1708e2fdb42 2024-11-13T00:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,550 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17d1d2a377a9417bb4edd1708e2fdb42, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T00:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,551 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 96d540c45076fbf64e35c80767cd2e16 in 468ms, sequenceid=49, compaction requested=false 2024-11-13T00:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T00:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T00:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T00:15:06,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-11-13T00:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0890 sec 2024-11-13T00:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:15:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111365a74b0be08b41b4a330a4e48ef0aadc_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_1/A:col10/1731456906216/Put/seqid=0 2024-11-13T00:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742220_1396 (size=12154) 2024-11-13T00:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T00:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,606 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T00:15:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456966668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456966669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456966670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456966673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456966674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:06,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:06,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456966786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456966786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456966786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456966787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456966787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:06,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:06,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:07,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456966995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456966996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456966996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456966996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456966996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111365a74b0be08b41b4a330a4e48ef0aadc_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111365a74b0be08b41b4a330a4e48ef0aadc_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:07,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/39e2cd14902f4778aab0ab6128df97ce, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:07,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/39e2cd14902f4778aab0ab6128df97ce is 175, key is test_row_1/A:col10/1731456906216/Put/seqid=0 2024-11-13T00:15:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742221_1397 (size=30951) 2024-11-13T00:15:07,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:07,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:07,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:07,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456967305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456967306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456967306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456967306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456967306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:07,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/39e2cd14902f4778aab0ab6128df97ce 2024-11-13T00:15:07,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/70a156cc4c284f7fbd65e7eb55b62dd7 is 50, key is test_row_1/B:col10/1731456906216/Put/seqid=0 2024-11-13T00:15:07,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742222_1398 (size=9657) 2024-11-13T00:15:07,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/70a156cc4c284f7fbd65e7eb55b62dd7 2024-11-13T00:15:07,424 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:15:07,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/cf90babcac564851b3dcac0f936a1eda is 50, key is test_row_1/C:col10/1731456906216/Put/seqid=0 2024-11-13T00:15:07,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742223_1399 (size=9657) 2024-11-13T00:15:07,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:07,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456967816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456967816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456967816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456967817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456967826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:07,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:07,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:07,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/cf90babcac564851b3dcac0f936a1eda 2024-11-13T00:15:07,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/39e2cd14902f4778aab0ab6128df97ce as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce 2024-11-13T00:15:07,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce, entries=150, sequenceid=61, filesize=30.2 K 2024-11-13T00:15:07,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/70a156cc4c284f7fbd65e7eb55b62dd7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/70a156cc4c284f7fbd65e7eb55b62dd7 2024-11-13T00:15:07,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/70a156cc4c284f7fbd65e7eb55b62dd7, entries=100, sequenceid=61, filesize=9.4 K 2024-11-13T00:15:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/cf90babcac564851b3dcac0f936a1eda as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/cf90babcac564851b3dcac0f936a1eda 2024-11-13T00:15:07,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/cf90babcac564851b3dcac0f936a1eda, entries=100, sequenceid=61, filesize=9.4 K 2024-11-13T00:15:07,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 96d540c45076fbf64e35c80767cd2e16 in 1278ms, sequenceid=61, compaction requested=true 2024-11-13T00:15:07,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:07,867 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:07,867 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:07,868 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,868 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,868 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/34307627197e4bd5b83aea31b0a11726, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4e6568e0bcde4263a481c7e6ad5e5342, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/70a156cc4c284f7fbd65e7eb55b62dd7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=32.9 K 2024-11-13T00:15:07,868 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=99.1 K 2024-11-13T00:15:07,868 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce] 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 34307627197e4bd5b83aea31b0a11726, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731456905470 2024-11-13T00:15:07,868 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df43ce7d604491096520075d316eb82, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731456905470 2024-11-13T00:15:07,869 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6568e0bcde4263a481c7e6ad5e5342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731456905549 2024-11-13T00:15:07,869 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6eb547d24249848c1f9cce2a45395f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731456905549 2024-11-13T00:15:07,869 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a156cc4c284f7fbd65e7eb55b62dd7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731456906216 2024-11-13T00:15:07,869 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39e2cd14902f4778aab0ab6128df97ce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731456906216 2024-11-13T00:15:07,876 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:07,878 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135fe13f40f15d4cdfa30fcd0da5908257_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:07,879 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:07,879 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/9bdfbe23f3614e2c93f6568f7a803f01 is 50, key is test_row_0/B:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:07,881 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135fe13f40f15d4cdfa30fcd0da5908257_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:07,882 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135fe13f40f15d4cdfa30fcd0da5908257_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:07,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742224_1400 (size=12104) 2024-11-13T00:15:07,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742225_1401 (size=4469) 2024-11-13T00:15:07,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T00:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:07,988 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:07,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:07,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135336ae4a198844e0a773b1476ca87d89_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456906672/Put/seqid=0 2024-11-13T00:15:08,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742226_1402 (size=12154) 2024-11-13T00:15:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:08,014 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135336ae4a198844e0a773b1476ca87d89_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135336ae4a198844e0a773b1476ca87d89_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:08,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/bab9da4df2d240ac8bc592970cd5ff3d, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:08,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/bab9da4df2d240ac8bc592970cd5ff3d is 175, key is test_row_0/A:col10/1731456906672/Put/seqid=0 2024-11-13T00:15:08,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742227_1403 (size=30955) 2024-11-13T00:15:08,292 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/9bdfbe23f3614e2c93f6568f7a803f01 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9bdfbe23f3614e2c93f6568f7a803f01 2024-11-13T00:15:08,295 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:08,295 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6362cce124054c0b924ca055e4416874 is 175, key is test_row_0/A:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:08,301 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into 9bdfbe23f3614e2c93f6568f7a803f01(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:08,301 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:08,301 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=13, startTime=1731456907867; duration=0sec 2024-11-13T00:15:08,301 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:08,301 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:08,301 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:08,302 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:08,302 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:08,302 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:08,302 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a7cadaee62174aea9405501781eaff0b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17d1d2a377a9417bb4edd1708e2fdb42, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/cf90babcac564851b3dcac0f936a1eda] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=32.9 K 2024-11-13T00:15:08,303 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a7cadaee62174aea9405501781eaff0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731456905470 2024-11-13T00:15:08,303 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 17d1d2a377a9417bb4edd1708e2fdb42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731456905549 2024-11-13T00:15:08,304 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting cf90babcac564851b3dcac0f936a1eda, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731456906216 2024-11-13T00:15:08,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742228_1404 (size=31165) 2024-11-13T00:15:08,328 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:08,329 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/fb966f04ec2645349c9d1eb61b60a1d4 is 50, key is test_row_0/C:col10/1731456905551/Put/seqid=0 2024-11-13T00:15:08,332 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6362cce124054c0b924ca055e4416874 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6362cce124054c0b924ca055e4416874 2024-11-13T00:15:08,337 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into 6362cce124054c0b924ca055e4416874(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:08,338 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:08,338 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=13, startTime=1731456907866; duration=0sec 2024-11-13T00:15:08,338 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:08,338 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:08,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742229_1405 (size=12104) 2024-11-13T00:15:08,423 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/bab9da4df2d240ac8bc592970cd5ff3d 2024-11-13T00:15:08,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/248b93fa79e5416883e0a5d1917a6782 is 50, key is test_row_0/B:col10/1731456906672/Put/seqid=0 2024-11-13T00:15:08,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742230_1406 (size=12001) 2024-11-13T00:15:08,747 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/fb966f04ec2645349c9d1eb61b60a1d4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/fb966f04ec2645349c9d1eb61b60a1d4 2024-11-13T00:15:08,754 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into fb966f04ec2645349c9d1eb61b60a1d4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:08,754 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:08,754 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=13, startTime=1731456907867; duration=0sec 2024-11-13T00:15:08,754 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:08,754 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:08,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:08,840 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/248b93fa79e5416883e0a5d1917a6782 2024-11-13T00:15:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/3f121e4f96e84b02afcc3ba30a87447a is 50, key is test_row_0/C:col10/1731456906672/Put/seqid=0 2024-11-13T00:15:08,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742231_1407 (size=12001) 2024-11-13T00:15:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456968844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456968853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456968855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456968855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456968855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456968965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456968966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456968976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456968976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456968976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456969176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456969176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456969186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456969186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456969186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,252 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/3f121e4f96e84b02afcc3ba30a87447a 2024-11-13T00:15:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/bab9da4df2d240ac8bc592970cd5ff3d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d 2024-11-13T00:15:09,260 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d, entries=150, sequenceid=81, filesize=30.2 K 2024-11-13T00:15:09,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/248b93fa79e5416883e0a5d1917a6782 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/248b93fa79e5416883e0a5d1917a6782 2024-11-13T00:15:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,265 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/248b93fa79e5416883e0a5d1917a6782, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T00:15:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/3f121e4f96e84b02afcc3ba30a87447a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/3f121e4f96e84b02afcc3ba30a87447a 2024-11-13T00:15:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,269 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/3f121e4f96e84b02afcc3ba30a87447a, entries=150, sequenceid=81, filesize=11.7 K 2024-11-13T00:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,270 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 96d540c45076fbf64e35c80767cd2e16 in 1282ms, sequenceid=81, compaction requested=false 2024-11-13T00:15:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T00:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T00:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T00:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6630 sec 2024-11-13T00:15:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.6720 sec 2024-11-13T00:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:15:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113753105ffcadf44dd858589120349d57c_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742232_1408 (size=12154) 2024-11-13T00:15:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113753105ffcadf44dd858589120349d57c_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113753105ffcadf44dd858589120349d57c_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a4fed389c6364b57b44347d607c506db, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a4fed389c6364b57b44347d607c506db is 175, key is test_row_0/A:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742233_1409 (size=30955) 2024-11-13T00:15:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a4fed389c6364b57b44347d607c506db 2024-11-13T00:15:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/aa157e1c3eea4aac827e3566ea765f22 is 50, key is test_row_0/B:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:09,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742234_1410 (size=12001) 2024-11-13T00:15:09,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/aa157e1c3eea4aac827e3566ea765f22 2024-11-13T00:15:09,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/d2f59b187917439a8bcf21318adc5b9c is 50, key is test_row_0/C:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:09,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456969558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456969563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742235_1411 (size=12001) 2024-11-13T00:15:09,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/d2f59b187917439a8bcf21318adc5b9c 2024-11-13T00:15:09,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a4fed389c6364b57b44347d607c506db as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db 2024-11-13T00:15:09,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db, entries=150, sequenceid=98, filesize=30.2 K 2024-11-13T00:15:09,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456969564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456969566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/aa157e1c3eea4aac827e3566ea765f22 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/aa157e1c3eea4aac827e3566ea765f22 2024-11-13T00:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456969570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/aa157e1c3eea4aac827e3566ea765f22, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T00:15:09,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/d2f59b187917439a8bcf21318adc5b9c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/d2f59b187917439a8bcf21318adc5b9c 2024-11-13T00:15:09,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/d2f59b187917439a8bcf21318adc5b9c, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T00:15:09,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 96d540c45076fbf64e35c80767cd2e16 in 80ms, sequenceid=98, compaction requested=true 2024-11-13T00:15:09,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:09,593 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:09,593 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:09,593 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:09,593 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:09,594 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:09,594 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:09,594 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:09,594 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:09,594 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6362cce124054c0b924ca055e4416874, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=90.9 K 2024-11-13T00:15:09,594 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:09,594 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9bdfbe23f3614e2c93f6568f7a803f01, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/248b93fa79e5416883e0a5d1917a6782, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/aa157e1c3eea4aac827e3566ea765f22] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=35.3 K 2024-11-13T00:15:09,594 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6362cce124054c0b924ca055e4416874, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db] 2024-11-13T00:15:09,597 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdfbe23f3614e2c93f6568f7a803f01, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731456905551 2024-11-13T00:15:09,597 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6362cce124054c0b924ca055e4416874, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731456905551 2024-11-13T00:15:09,598 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 248b93fa79e5416883e0a5d1917a6782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731456906668 2024-11-13T00:15:09,598 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab9da4df2d240ac8bc592970cd5ff3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731456906668 2024-11-13T00:15:09,598 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting aa157e1c3eea4aac827e3566ea765f22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731456908841 2024-11-13T00:15:09,599 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4fed389c6364b57b44347d607c506db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731456908841 2024-11-13T00:15:09,605 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:09,608 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d900ed82f1be444da7216e7632160e47_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:09,609 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:09,610 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/56e562148cf04b129553048b882694f7 is 50, key is test_row_0/B:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:09,610 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d900ed82f1be444da7216e7632160e47_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:09,610 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d900ed82f1be444da7216e7632160e47_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742236_1412 (size=12207) 2024-11-13T00:15:09,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742237_1413 (size=4469) 2024-11-13T00:15:09,628 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#348 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:09,629 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/1ea98089b44d49ba9f816b93e430970d is 175, key is test_row_0/A:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:09,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742238_1414 (size=31161) 2024-11-13T00:15:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:09,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:15:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:09,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113144f5b1b2615480bb7a53e0a8d1f9ce6_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456909567/Put/seqid=0 2024-11-13T00:15:09,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742239_1415 (size=14594) 2024-11-13T00:15:09,696 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:09,701 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113144f5b1b2615480bb7a53e0a8d1f9ce6_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113144f5b1b2615480bb7a53e0a8d1f9ce6_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:09,702 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/df8c11e85c1946a7a6879da57c0baaec, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:09,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/df8c11e85c1946a7a6879da57c0baaec is 175, key is test_row_0/A:col10/1731456909567/Put/seqid=0 2024-11-13T00:15:09,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742240_1416 (size=39549) 2024-11-13T00:15:09,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456969699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456969711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456969713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456969714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456969715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456969825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456969825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456969836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456969836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456969836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,024 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/56e562148cf04b129553048b882694f7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/56e562148cf04b129553048b882694f7 2024-11-13T00:15:10,029 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into 56e562148cf04b129553048b882694f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:10,029 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:10,029 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=13, startTime=1731456909593; duration=0sec 2024-11-13T00:15:10,029 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:10,029 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:10,029 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:10,030 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:10,030 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:10,030 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:10,030 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/fb966f04ec2645349c9d1eb61b60a1d4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/3f121e4f96e84b02afcc3ba30a87447a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/d2f59b187917439a8bcf21318adc5b9c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=35.3 K 2024-11-13T00:15:10,030 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting fb966f04ec2645349c9d1eb61b60a1d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731456905551 2024-11-13T00:15:10,031 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f121e4f96e84b02afcc3ba30a87447a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731456906668 2024-11-13T00:15:10,031 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f59b187917439a8bcf21318adc5b9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731456908841 2024-11-13T00:15:10,040 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:10,040 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/0d7349de4e1e4ddcaa0720329b979ed8 is 50, key is test_row_0/C:col10/1731456908841/Put/seqid=0 2024-11-13T00:15:10,040 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/1ea98089b44d49ba9f816b93e430970d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/1ea98089b44d49ba9f816b93e430970d 2024-11-13T00:15:10,045 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into 1ea98089b44d49ba9f816b93e430970d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:10,045 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:10,045 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=13, startTime=1731456909592; duration=0sec 2024-11-13T00:15:10,046 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:10,046 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:10,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456970046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456970046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742241_1417 (size=12207) 2024-11-13T00:15:10,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456970056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456970056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456970056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,105 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/df8c11e85c1946a7a6879da57c0baaec 2024-11-13T00:15:10,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/3299379720c54dfe93f8f8811d9a8314 is 50, key is test_row_0/B:col10/1731456909567/Put/seqid=0 2024-11-13T00:15:10,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742242_1418 (size=12001) 2024-11-13T00:15:10,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456970356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456970356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456970376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456970376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456970376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,460 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/0d7349de4e1e4ddcaa0720329b979ed8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/0d7349de4e1e4ddcaa0720329b979ed8 2024-11-13T00:15:10,464 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into 0d7349de4e1e4ddcaa0720329b979ed8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:10,464 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:10,464 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=13, startTime=1731456909593; duration=0sec 2024-11-13T00:15:10,464 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:10,464 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:10,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/3299379720c54dfe93f8f8811d9a8314 2024-11-13T00:15:10,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/38e3c6582ba44ea48f5d46aa2174e59a is 50, key is test_row_0/C:col10/1731456909567/Put/seqid=0 2024-11-13T00:15:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742243_1419 (size=12001) 2024-11-13T00:15:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T00:15:10,777 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:10,777 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T00:15:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T00:15:10,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:10,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:10,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:10,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456970876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456970876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T00:15:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456970885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456970886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456970896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:10,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/38e3c6582ba44ea48f5d46aa2174e59a 2024-11-13T00:15:10,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/df8c11e85c1946a7a6879da57c0baaec as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec 2024-11-13T00:15:10,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T00:15:10,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:10,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:10,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec, entries=200, sequenceid=117, filesize=38.6 K 2024-11-13T00:15:10,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/3299379720c54dfe93f8f8811d9a8314 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3299379720c54dfe93f8f8811d9a8314 2024-11-13T00:15:10,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3299379720c54dfe93f8f8811d9a8314, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T00:15:10,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/38e3c6582ba44ea48f5d46aa2174e59a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/38e3c6582ba44ea48f5d46aa2174e59a 2024-11-13T00:15:10,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/38e3c6582ba44ea48f5d46aa2174e59a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T00:15:10,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 96d540c45076fbf64e35c80767cd2e16 in 1265ms, sequenceid=117, compaction requested=false 2024-11-13T00:15:10,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:11,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T00:15:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:11,086 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:15:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:11,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371f046fd225a4231a69100a7d9cbea9d_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456909713/Put/seqid=0 2024-11-13T00:15:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T00:15:11,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742244_1420 (size=12204) 2024-11-13T00:15:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T00:15:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:11,504 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371f046fd225a4231a69100a7d9cbea9d_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371f046fd225a4231a69100a7d9cbea9d_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:11,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/da0249572e0740578c71ac57bb27eb7f, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:11,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/da0249572e0740578c71ac57bb27eb7f is 175, key is test_row_0/A:col10/1731456909713/Put/seqid=0 2024-11-13T00:15:11,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742245_1421 (size=31005) 2024-11-13T00:15:11,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/da0249572e0740578c71ac57bb27eb7f 2024-11-13T00:15:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T00:15:11,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/a93c37da76644892b92e62c089daf4b8 is 50, key is test_row_0/B:col10/1731456909713/Put/seqid=0 2024-11-13T00:15:11,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742246_1422 (size=12051) 2024-11-13T00:15:11,967 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/a93c37da76644892b92e62c089daf4b8 2024-11-13T00:15:11,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456971951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:11,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:11,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456971956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456971960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:11,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456971968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:11,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456971970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/17de4788f1a746729d8b1acb54fb7b66 is 50, key is test_row_0/C:col10/1731456909713/Put/seqid=0 2024-11-13T00:15:12,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742247_1423 (size=12051) 2024-11-13T00:15:12,030 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/17de4788f1a746729d8b1acb54fb7b66 2024-11-13T00:15:12,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/da0249572e0740578c71ac57bb27eb7f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f 2024-11-13T00:15:12,049 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f, entries=150, sequenceid=133, filesize=30.3 K 2024-11-13T00:15:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/a93c37da76644892b92e62c089daf4b8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/a93c37da76644892b92e62c089daf4b8 2024-11-13T00:15:12,060 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/a93c37da76644892b92e62c089daf4b8, entries=150, sequenceid=133, filesize=11.8 K 2024-11-13T00:15:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/17de4788f1a746729d8b1acb54fb7b66 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17de4788f1a746729d8b1acb54fb7b66 2024-11-13T00:15:12,076 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17de4788f1a746729d8b1acb54fb7b66, entries=150, sequenceid=133, filesize=11.8 K 2024-11-13T00:15:12,078 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 96d540c45076fbf64e35c80767cd2e16 in 992ms, sequenceid=133, compaction requested=true 2024-11-13T00:15:12,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:12,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:12,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T00:15:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T00:15:12,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T00:15:12,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3000 sec 2024-11-13T00:15:12,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.3050 sec 2024-11-13T00:15:12,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:15:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:12,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:12,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137342276210da435ca32978127598717a_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456972115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456972117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456972118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456972121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456972124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742248_1424 (size=14794) 2024-11-13T00:15:12,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:12,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137342276210da435ca32978127598717a_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137342276210da435ca32978127598717a_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:12,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a7f2118cba85438b851cd5b455c35f91, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:12,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a7f2118cba85438b851cd5b455c35f91 is 175, key is test_row_0/A:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742249_1425 (size=39749) 2024-11-13T00:15:12,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a7f2118cba85438b851cd5b455c35f91 2024-11-13T00:15:12,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/be33284d88fb4e8b91aa5c2a8d4670a1 is 50, key is test_row_0/B:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456972246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456972246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456972246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456972246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456972246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742250_1426 (size=12151) 2024-11-13T00:15:12,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/be33284d88fb4e8b91aa5c2a8d4670a1 2024-11-13T00:15:12,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/1766d2e8cce7461d8dcb11e1fab3b663 is 50, key is test_row_0/C:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742251_1427 (size=12151) 2024-11-13T00:15:12,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/1766d2e8cce7461d8dcb11e1fab3b663 2024-11-13T00:15:12,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/a7f2118cba85438b851cd5b455c35f91 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91 2024-11-13T00:15:12,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91, entries=200, sequenceid=153, filesize=38.8 K 2024-11-13T00:15:12,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/be33284d88fb4e8b91aa5c2a8d4670a1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/be33284d88fb4e8b91aa5c2a8d4670a1 2024-11-13T00:15:12,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/be33284d88fb4e8b91aa5c2a8d4670a1, entries=150, sequenceid=153, filesize=11.9 K 2024-11-13T00:15:12,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/1766d2e8cce7461d8dcb11e1fab3b663 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1766d2e8cce7461d8dcb11e1fab3b663 2024-11-13T00:15:12,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1766d2e8cce7461d8dcb11e1fab3b663, entries=150, sequenceid=153, filesize=11.9 K 2024-11-13T00:15:12,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 96d540c45076fbf64e35c80767cd2e16 in 258ms, sequenceid=153, compaction requested=true 2024-11-13T00:15:12,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:12,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:12,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:12,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:12,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:12,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:12,347 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:12,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T00:15:12,347 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:12,348 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:12,348 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:12,348 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:12,348 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/1ea98089b44d49ba9f816b93e430970d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=138.1 K 2024-11-13T00:15:12,348 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:12,349 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/1ea98089b44d49ba9f816b93e430970d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91] 2024-11-13T00:15:12,349 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:12,349 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:12,349 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea98089b44d49ba9f816b93e430970d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731456908841 2024-11-13T00:15:12,349 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:12,349 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/56e562148cf04b129553048b882694f7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3299379720c54dfe93f8f8811d9a8314, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/a93c37da76644892b92e62c089daf4b8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/be33284d88fb4e8b91aa5c2a8d4670a1] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=47.3 K 2024-11-13T00:15:12,349 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56e562148cf04b129553048b882694f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731456908841 2024-11-13T00:15:12,349 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting df8c11e85c1946a7a6879da57c0baaec, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731456909543 2024-11-13T00:15:12,350 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3299379720c54dfe93f8f8811d9a8314, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731456909543 2024-11-13T00:15:12,350 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting da0249572e0740578c71ac57bb27eb7f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731456909696 2024-11-13T00:15:12,350 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93c37da76644892b92e62c089daf4b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731456909696 2024-11-13T00:15:12,350 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f2118cba85438b851cd5b455c35f91, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456911933 2024-11-13T00:15:12,350 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting be33284d88fb4e8b91aa5c2a8d4670a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456911933 2024-11-13T00:15:12,373 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:12,373 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/3d019019579a43848706f05070647f7b is 50, key is test_row_0/B:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,384 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:12,418 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111323dc497ce24e449d9b6869ce47340d0c_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:12,421 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111323dc497ce24e449d9b6869ce47340d0c_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:12,421 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323dc497ce24e449d9b6869ce47340d0c_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:12,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742252_1428 (size=12493) 2024-11-13T00:15:12,452 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/3d019019579a43848706f05070647f7b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3d019019579a43848706f05070647f7b 2024-11-13T00:15:12,459 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into 3d019019579a43848706f05070647f7b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:12,459 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:12,459 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=12, startTime=1731456912347; duration=0sec 2024-11-13T00:15:12,459 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:12,459 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:12,459 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:12,462 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:12,462 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:12,462 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:12,463 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/0d7349de4e1e4ddcaa0720329b979ed8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/38e3c6582ba44ea48f5d46aa2174e59a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17de4788f1a746729d8b1acb54fb7b66, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1766d2e8cce7461d8dcb11e1fab3b663] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=47.3 K 2024-11-13T00:15:12,463 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7349de4e1e4ddcaa0720329b979ed8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731456908841 2024-11-13T00:15:12,463 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38e3c6582ba44ea48f5d46aa2174e59a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731456909543 2024-11-13T00:15:12,464 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17de4788f1a746729d8b1acb54fb7b66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731456909696 2024-11-13T00:15:12,464 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1766d2e8cce7461d8dcb11e1fab3b663, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456911933 2024-11-13T00:15:12,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742253_1429 (size=4469) 2024-11-13T00:15:12,470 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#361 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:12,471 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/238b0ec0c47e48968aeb7848ebf41de6 is 175, key is test_row_0/A:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:12,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T00:15:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:12,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742254_1430 (size=31447) 2024-11-13T00:15:12,495 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:12,496 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/994468fb14c54ef9bd14ebcabba1f058 is 50, key is test_row_0/C:col10/1731456911968/Put/seqid=0 2024-11-13T00:15:12,506 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/238b0ec0c47e48968aeb7848ebf41de6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/238b0ec0c47e48968aeb7848ebf41de6 2024-11-13T00:15:12,510 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into 238b0ec0c47e48968aeb7848ebf41de6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:12,510 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:12,510 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=12, startTime=1731456912346; duration=0sec 2024-11-13T00:15:12,510 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:12,510 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:12,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d825f5f6ac094a19b4c6f1a35bf60ab1_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456912114/Put/seqid=0 2024-11-13T00:15:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456972501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456972503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742255_1431 (size=12493) 2024-11-13T00:15:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456972515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456972516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456972517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742256_1432 (size=14794) 2024-11-13T00:15:12,556 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/994468fb14c54ef9bd14ebcabba1f058 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/994468fb14c54ef9bd14ebcabba1f058 2024-11-13T00:15:12,571 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into 994468fb14c54ef9bd14ebcabba1f058(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:12,571 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:12,571 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=12, startTime=1731456912347; duration=0sec 2024-11-13T00:15:12,572 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:12,572 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456972626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456972626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456972636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456972636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456972636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456972845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456972846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456972856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456972856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456972856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T00:15:12,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:12,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T00:15:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T00:15:12,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:12,940 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:12,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:12,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:12,961 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d825f5f6ac094a19b4c6f1a35bf60ab1_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d825f5f6ac094a19b4c6f1a35bf60ab1_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:12,962 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/c1cc9c0342b642aba171f9d223052b3e, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:12,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/c1cc9c0342b642aba171f9d223052b3e is 175, key is test_row_0/A:col10/1731456912114/Put/seqid=0 2024-11-13T00:15:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742257_1433 (size=39749) 2024-11-13T00:15:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T00:15:13,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T00:15:13,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456973156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456973166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456973175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456973176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456973176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T00:15:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T00:15:13,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/c1cc9c0342b642aba171f9d223052b3e 2024-11-13T00:15:13,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/0bb57e0fbced48688a0d2b5bba94568b is 50, key is test_row_0/B:col10/1731456912114/Put/seqid=0 2024-11-13T00:15:13,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742258_1434 (size=12151) 2024-11-13T00:15:13,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/0bb57e0fbced48688a0d2b5bba94568b 2024-11-13T00:15:13,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/150a3727d0c047b6950a583edc19ace7 is 50, key is test_row_0/C:col10/1731456912114/Put/seqid=0 2024-11-13T00:15:13,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742259_1435 (size=12151) 2024-11-13T00:15:13,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T00:15:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T00:15:13,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:13,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T00:15:13,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456973676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456973686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456973686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456973686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456973687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:13,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T00:15:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:13,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/150a3727d0c047b6950a583edc19ace7 2024-11-13T00:15:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/c1cc9c0342b642aba171f9d223052b3e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e 2024-11-13T00:15:13,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e, entries=200, sequenceid=167, filesize=38.8 K 2024-11-13T00:15:13,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/0bb57e0fbced48688a0d2b5bba94568b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/0bb57e0fbced48688a0d2b5bba94568b 2024-11-13T00:15:13,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/0bb57e0fbced48688a0d2b5bba94568b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T00:15:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/150a3727d0c047b6950a583edc19ace7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/150a3727d0c047b6950a583edc19ace7 2024-11-13T00:15:13,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/150a3727d0c047b6950a583edc19ace7, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T00:15:13,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 96d540c45076fbf64e35c80767cd2e16 in 1334ms, sequenceid=167, compaction requested=false 2024-11-13T00:15:13,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:13,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T00:15:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:13,861 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:15:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113946e0dd6723d42dfa3d543f813935af8_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456912512/Put/seqid=0 2024-11-13T00:15:13,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742260_1436 (size=12304) 2024-11-13T00:15:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T00:15:14,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:14,278 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113946e0dd6723d42dfa3d543f813935af8_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113946e0dd6723d42dfa3d543f813935af8_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:14,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/f12386b923c942d5ba9dafc8e5c91759, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/f12386b923c942d5ba9dafc8e5c91759 is 175, key is test_row_0/A:col10/1731456912512/Put/seqid=0 2024-11-13T00:15:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742261_1437 (size=31105) 2024-11-13T00:15:14,683 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/f12386b923c942d5ba9dafc8e5c91759 2024-11-13T00:15:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/9b240db90c43407296e80ab01ec69e05 is 50, key is test_row_0/B:col10/1731456912512/Put/seqid=0 2024-11-13T00:15:14,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742262_1438 (size=12151) 2024-11-13T00:15:14,702 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/9b240db90c43407296e80ab01ec69e05 2024-11-13T00:15:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/a66a0854e118442c87078d7996a3de0e is 50, key is test_row_0/C:col10/1731456912512/Put/seqid=0 2024-11-13T00:15:14,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742263_1439 (size=12151) 2024-11-13T00:15:14,714 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/a66a0854e118442c87078d7996a3de0e 2024-11-13T00:15:14,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/f12386b923c942d5ba9dafc8e5c91759 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759 2024-11-13T00:15:14,723 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759, entries=150, sequenceid=188, filesize=30.4 K 2024-11-13T00:15:14,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/9b240db90c43407296e80ab01ec69e05 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9b240db90c43407296e80ab01ec69e05 2024-11-13T00:15:14,728 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9b240db90c43407296e80ab01ec69e05, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T00:15:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/a66a0854e118442c87078d7996a3de0e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a66a0854e118442c87078d7996a3de0e 2024-11-13T00:15:14,732 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a66a0854e118442c87078d7996a3de0e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T00:15:14,733 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=53.67 KB/54960 for 96d540c45076fbf64e35c80767cd2e16 in 872ms, sequenceid=188, compaction requested=true 2024-11-13T00:15:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T00:15:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:14,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:15:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T00:15:14,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:14,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:14,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:14,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:14,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:14,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:14,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T00:15:14,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7950 sec 2024-11-13T00:15:14,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8010 sec 2024-11-13T00:15:14,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137439ddaf30eb465c837fbb08e1ed9f91_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:14,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742264_1440 (size=17284) 2024-11-13T00:15:14,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456974773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456974775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456974776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456974777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456974779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456974886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456974896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456974896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456974896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456974896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T00:15:15,086 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:15,087 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T00:15:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T00:15:15,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:15,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:15,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:15,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456975096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456975116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456975116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456975117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456975117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,149 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:15,152 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137439ddaf30eb465c837fbb08e1ed9f91_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137439ddaf30eb465c837fbb08e1ed9f91_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:15,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/51dc3c00cbf14fc2b27030c30f850430, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:15,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/51dc3c00cbf14fc2b27030c30f850430 is 175, key is test_row_0/A:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:15,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742265_1441 (size=48389) 2024-11-13T00:15:15,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/51dc3c00cbf14fc2b27030c30f850430 2024-11-13T00:15:15,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/7a964ec04d5f47ef93634f347eb4e802 is 50, key is test_row_0/B:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742266_1442 (size=12151) 2024-11-13T00:15:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T00:15:15,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T00:15:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T00:15:15,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:15,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T00:15:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456975406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456975436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456975436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456975436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456975436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T00:15:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:15,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/7a964ec04d5f47ef93634f347eb4e802 2024-11-13T00:15:15,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/9a5eb419c5d4482882ff3c6a29f8eac0 is 50, key is test_row_0/C:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:15,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742267_1443 (size=12151) 2024-11-13T00:15:15,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/9a5eb419c5d4482882ff3c6a29f8eac0 2024-11-13T00:15:15,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/51dc3c00cbf14fc2b27030c30f850430 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430 2024-11-13T00:15:15,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430, entries=250, sequenceid=201, filesize=47.3 K 2024-11-13T00:15:15,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/7a964ec04d5f47ef93634f347eb4e802 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/7a964ec04d5f47ef93634f347eb4e802 2024-11-13T00:15:15,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/7a964ec04d5f47ef93634f347eb4e802, entries=150, sequenceid=201, filesize=11.9 K 2024-11-13T00:15:15,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/9a5eb419c5d4482882ff3c6a29f8eac0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/9a5eb419c5d4482882ff3c6a29f8eac0 2024-11-13T00:15:15,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/9a5eb419c5d4482882ff3c6a29f8eac0, entries=150, sequenceid=201, filesize=11.9 K 2024-11-13T00:15:15,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 96d540c45076fbf64e35c80767cd2e16 in 868ms, sequenceid=201, compaction requested=true 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:15,602 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:15,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:15,602 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150690 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:15,604 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,604 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,604 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3d019019579a43848706f05070647f7b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/0bb57e0fbced48688a0d2b5bba94568b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9b240db90c43407296e80ab01ec69e05, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/7a964ec04d5f47ef93634f347eb4e802] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=47.8 K 2024-11-13T00:15:15,604 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/238b0ec0c47e48968aeb7848ebf41de6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=147.2 K 2024-11-13T00:15:15,604 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/238b0ec0c47e48968aeb7848ebf41de6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430] 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d019019579a43848706f05070647f7b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456911933 2024-11-13T00:15:15,604 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 238b0ec0c47e48968aeb7848ebf41de6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456911933 2024-11-13T00:15:15,605 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb57e0fbced48688a0d2b5bba94568b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731456912114 2024-11-13T00:15:15,605 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1cc9c0342b642aba171f9d223052b3e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731456912114 2024-11-13T00:15:15,605 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b240db90c43407296e80ab01ec69e05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731456912494 2024-11-13T00:15:15,605 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12386b923c942d5ba9dafc8e5c91759, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731456912494 2024-11-13T00:15:15,605 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a964ec04d5f47ef93634f347eb4e802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731456914717 2024-11-13T00:15:15,605 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51dc3c00cbf14fc2b27030c30f850430, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731456914717 2024-11-13T00:15:15,614 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:15,616 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:15,616 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d69b571110094646844ee7651c876084 is 50, key is test_row_0/B:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:15,617 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113beaaee2dbaf6426bb09978e4cc3a28df_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:15,620 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113beaaee2dbaf6426bb09978e4cc3a28df_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:15,620 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113beaaee2dbaf6426bb09978e4cc3a28df_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742268_1444 (size=12629) 2024-11-13T00:15:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742269_1445 (size=4469) 2024-11-13T00:15:15,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:15,701 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:15,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111313ab1a54b44c40f2954816160b0c87e6_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456914776/Put/seqid=0 2024-11-13T00:15:15,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742270_1446 (size=12304) 2024-11-13T00:15:15,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T00:15:15,717 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111313ab1a54b44c40f2954816160b0c87e6_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111313ab1a54b44c40f2954816160b0c87e6_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/edfbcef2a12d4ede946cee323eb38df8, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/edfbcef2a12d4ede946cee323eb38df8 is 175, key is test_row_0/A:col10/1731456914776/Put/seqid=0 2024-11-13T00:15:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742271_1447 (size=31105) 2024-11-13T00:15:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:15,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456975956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456975957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456975963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456975967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456975968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,027 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d69b571110094646844ee7651c876084 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d69b571110094646844ee7651c876084 2024-11-13T00:15:16,028 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#373 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:16,028 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/47b06864cfa747ea819316ad19d1c636 is 175, key is test_row_0/A:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:16,031 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into d69b571110094646844ee7651c876084(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:16,032 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:16,032 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=12, startTime=1731456915602; duration=0sec 2024-11-13T00:15:16,032 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:16,032 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:16,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742272_1448 (size=31583) 2024-11-13T00:15:16,032 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:16,033 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:16,033 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:16,034 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:16,034 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/994468fb14c54ef9bd14ebcabba1f058, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/150a3727d0c047b6950a583edc19ace7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a66a0854e118442c87078d7996a3de0e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/9a5eb419c5d4482882ff3c6a29f8eac0] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=47.8 K 2024-11-13T00:15:16,034 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 994468fb14c54ef9bd14ebcabba1f058, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731456911933 2024-11-13T00:15:16,035 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 150a3727d0c047b6950a583edc19ace7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731456912114 2024-11-13T00:15:16,035 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a66a0854e118442c87078d7996a3de0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731456912494 2024-11-13T00:15:16,035 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5eb419c5d4482882ff3c6a29f8eac0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731456914717 2024-11-13T00:15:16,037 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/47b06864cfa747ea819316ad19d1c636 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/47b06864cfa747ea819316ad19d1c636 2024-11-13T00:15:16,041 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into 47b06864cfa747ea819316ad19d1c636(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:16,041 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:16,041 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=12, startTime=1731456915602; duration=0sec 2024-11-13T00:15:16,041 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:16,041 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:16,047 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:16,047 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/808e1239118e4f61b016265744e78e33 is 50, key is test_row_0/C:col10/1731456914732/Put/seqid=0 2024-11-13T00:15:16,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742273_1449 (size=12629) 2024-11-13T00:15:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456976076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456976076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456976076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456976086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456976086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,123 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/edfbcef2a12d4ede946cee323eb38df8 2024-11-13T00:15:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d71f5c91111c473dabbb241662bb4139 is 50, key is test_row_0/B:col10/1731456914776/Put/seqid=0 2024-11-13T00:15:16,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742274_1450 (size=12151) 2024-11-13T00:15:16,145 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d71f5c91111c473dabbb241662bb4139 2024-11-13T00:15:16,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/1b9dfb8826984804b2f5c317e0a0f919 is 50, key is test_row_0/C:col10/1731456914776/Put/seqid=0 2024-11-13T00:15:16,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742275_1451 (size=12151) 2024-11-13T00:15:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T00:15:16,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456976295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456976296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456976296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456976306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456976306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,457 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/808e1239118e4f61b016265744e78e33 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/808e1239118e4f61b016265744e78e33 2024-11-13T00:15:16,461 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into 808e1239118e4f61b016265744e78e33(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:16,461 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:16,461 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=12, startTime=1731456915602; duration=0sec 2024-11-13T00:15:16,461 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:16,461 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:16,560 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/1b9dfb8826984804b2f5c317e0a0f919 2024-11-13T00:15:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/edfbcef2a12d4ede946cee323eb38df8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8 2024-11-13T00:15:16,569 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8, entries=150, sequenceid=220, filesize=30.4 K 2024-11-13T00:15:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d71f5c91111c473dabbb241662bb4139 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d71f5c91111c473dabbb241662bb4139 2024-11-13T00:15:16,574 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d71f5c91111c473dabbb241662bb4139, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T00:15:16,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/1b9dfb8826984804b2f5c317e0a0f919 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1b9dfb8826984804b2f5c317e0a0f919 2024-11-13T00:15:16,579 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1b9dfb8826984804b2f5c317e0a0f919, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T00:15:16,580 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 96d540c45076fbf64e35c80767cd2e16 in 879ms, sequenceid=220, compaction requested=false 2024-11-13T00:15:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T00:15:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T00:15:16,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T00:15:16,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2024-11-13T00:15:16,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4970 sec 2024-11-13T00:15:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:16,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:16,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111325393768d19a4fe699fd656db9865c6c_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:16,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742276_1452 (size=12304) 2024-11-13T00:15:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456976666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456976680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456976681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456976683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456976684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456976786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456976796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456976806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456976806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456976806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456977006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456977016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456977026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456977026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456977026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111325393768d19a4fe699fd656db9865c6c_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111325393768d19a4fe699fd656db9865c6c_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:17,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6a76c284785543379ac3e4131d77ca81, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:17,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6a76c284785543379ac3e4131d77ca81 is 175, key is test_row_0/A:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:17,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742277_1453 (size=31105) 2024-11-13T00:15:17,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6a76c284785543379ac3e4131d77ca81 2024-11-13T00:15:17,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/2a9318bd9967436098cee8ff05f27fa7 is 50, key is test_row_0/B:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742278_1454 (size=12151) 2024-11-13T00:15:17,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/2a9318bd9967436098cee8ff05f27fa7 2024-11-13T00:15:17,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/911e707f75934d61b555fd80337eeeeb is 50, key is test_row_0/C:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:17,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742279_1455 (size=12151) 2024-11-13T00:15:17,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/911e707f75934d61b555fd80337eeeeb 2024-11-13T00:15:17,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/6a76c284785543379ac3e4131d77ca81 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81 2024-11-13T00:15:17,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81, entries=150, sequenceid=238, filesize=30.4 K 2024-11-13T00:15:17,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/2a9318bd9967436098cee8ff05f27fa7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/2a9318bd9967436098cee8ff05f27fa7 2024-11-13T00:15:17,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/2a9318bd9967436098cee8ff05f27fa7, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T00:15:17,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/911e707f75934d61b555fd80337eeeeb as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/911e707f75934d61b555fd80337eeeeb 2024-11-13T00:15:17,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/911e707f75934d61b555fd80337eeeeb, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T00:15:17,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 96d540c45076fbf64e35c80767cd2e16 in 517ms, sequenceid=238, compaction requested=true 2024-11-13T00:15:17,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:17,140 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:17,141 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:17,141 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:17,141 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,141 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/47b06864cfa747ea819316ad19d1c636, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=91.6 K 2024-11-13T00:15:17,141 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,141 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/47b06864cfa747ea819316ad19d1c636, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81] 2024-11-13T00:15:17,142 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47b06864cfa747ea819316ad19d1c636, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731456914717 2024-11-13T00:15:17,142 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting edfbcef2a12d4ede946cee323eb38df8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731456914773 2024-11-13T00:15:17,142 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a76c284785543379ac3e4131d77ca81, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731456915953 2024-11-13T00:15:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:17,147 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:17,148 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:17,148 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:17,148 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,149 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d69b571110094646844ee7651c876084, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d71f5c91111c473dabbb241662bb4139, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/2a9318bd9967436098cee8ff05f27fa7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=36.1 K 2024-11-13T00:15:17,149 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d69b571110094646844ee7651c876084, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731456914717 2024-11-13T00:15:17,149 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d71f5c91111c473dabbb241662bb4139, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731456914773 2024-11-13T00:15:17,150 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a9318bd9967436098cee8ff05f27fa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731456915953 2024-11-13T00:15:17,153 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:17,157 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111331a17c754f6b4347b851dcbd83d46fdb_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:17,157 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:17,157 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/78dc5d91be184a36bf6ddbc81dd293d5 is 50, key is test_row_0/B:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:17,159 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111331a17c754f6b4347b851dcbd83d46fdb_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:17,159 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111331a17c754f6b4347b851dcbd83d46fdb_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:17,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742280_1456 (size=12731) 2024-11-13T00:15:17,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742281_1457 (size=4469) 2024-11-13T00:15:17,183 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#381 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:17,183 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/0385494c9f82404e8647b6171f3f0cf0 is 175, key is test_row_0/A:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:17,184 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/78dc5d91be184a36bf6ddbc81dd293d5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/78dc5d91be184a36bf6ddbc81dd293d5 2024-11-13T00:15:17,189 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into 78dc5d91be184a36bf6ddbc81dd293d5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:17,189 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:17,189 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=13, startTime=1731456917147; duration=0sec 2024-11-13T00:15:17,189 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:17,189 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:17,189 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:17,190 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:17,190 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:17,190 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,190 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/808e1239118e4f61b016265744e78e33, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1b9dfb8826984804b2f5c317e0a0f919, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/911e707f75934d61b555fd80337eeeeb] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=36.1 K 2024-11-13T00:15:17,190 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 808e1239118e4f61b016265744e78e33, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731456914717 2024-11-13T00:15:17,190 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9dfb8826984804b2f5c317e0a0f919, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731456914773 2024-11-13T00:15:17,191 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 911e707f75934d61b555fd80337eeeeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731456915953 2024-11-13T00:15:17,202 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:17,202 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/769a5fe262b84b8ba4cbf91298cf1ca3 is 50, key is test_row_0/C:col10/1731456916622/Put/seqid=0 2024-11-13T00:15:17,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742282_1458 (size=31685) 2024-11-13T00:15:17,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742283_1459 (size=12731) 2024-11-13T00:15:17,222 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/769a5fe262b84b8ba4cbf91298cf1ca3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/769a5fe262b84b8ba4cbf91298cf1ca3 2024-11-13T00:15:17,226 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into 769a5fe262b84b8ba4cbf91298cf1ca3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:17,226 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:17,226 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=13, startTime=1731456917153; duration=0sec 2024-11-13T00:15:17,226 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:17,226 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T00:15:17,236 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:17,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T00:15:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:17,239 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:17,239 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:17,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:17,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T00:15:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:17,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136732ca455ec3427fb13354581143ac4a_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456916682/Put/seqid=0 2024-11-13T00:15:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:17,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742284_1460 (size=14844) 2024-11-13T00:15:17,351 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:17,356 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136732ca455ec3427fb13354581143ac4a_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136732ca455ec3427fb13354581143ac4a_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:17,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/4d796653e7fd4c408ee8007c38c471ae, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:17,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/4d796653e7fd4c408ee8007c38c471ae is 175, key is test_row_0/A:col10/1731456916682/Put/seqid=0 2024-11-13T00:15:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742285_1461 (size=39799) 2024-11-13T00:15:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456977359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456977362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456977361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456977364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456977365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456977486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456977486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456977487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456977487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456977487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:17,608 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/0385494c9f82404e8647b6171f3f0cf0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/0385494c9f82404e8647b6171f3f0cf0 2024-11-13T00:15:17,613 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into 0385494c9f82404e8647b6171f3f0cf0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:17,613 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:17,613 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=13, startTime=1731456917140; duration=0sec 2024-11-13T00:15:17,613 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:17,613 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:17,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456977695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456977695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456977696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456977706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456977706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:17,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/4d796653e7fd4c408ee8007c38c471ae 2024-11-13T00:15:17,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d5e2fc7d4ebc4113845628fe913bb6e8 is 50, key is test_row_0/B:col10/1731456916682/Put/seqid=0 2024-11-13T00:15:17,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742286_1462 (size=12201) 2024-11-13T00:15:17,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:17,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:18,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456978016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456978017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456978017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456978017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456978017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d5e2fc7d4ebc4113845628fe913bb6e8 2024-11-13T00:15:18,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/8232143e26ce49e8a3454f794597250a is 50, key is test_row_0/C:col10/1731456916682/Put/seqid=0 2024-11-13T00:15:18,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742287_1463 (size=12201) 2024-11-13T00:15:18,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:18,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:18,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456978526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456978526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456978536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456978536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456978536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:18,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/8232143e26ce49e8a3454f794597250a 2024-11-13T00:15:18,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/4d796653e7fd4c408ee8007c38c471ae as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae 2024-11-13T00:15:18,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae, entries=200, sequenceid=258, filesize=38.9 K 2024-11-13T00:15:18,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d5e2fc7d4ebc4113845628fe913bb6e8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d5e2fc7d4ebc4113845628fe913bb6e8 2024-11-13T00:15:18,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d5e2fc7d4ebc4113845628fe913bb6e8, entries=150, sequenceid=258, filesize=11.9 K 2024-11-13T00:15:18,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/8232143e26ce49e8a3454f794597250a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8232143e26ce49e8a3454f794597250a 2024-11-13T00:15:18,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8232143e26ce49e8a3454f794597250a, entries=150, sequenceid=258, filesize=11.9 K 2024-11-13T00:15:18,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 96d540c45076fbf64e35c80767cd2e16 in 1283ms, sequenceid=258, compaction requested=false 2024-11-13T00:15:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:18,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T00:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:18,618 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T00:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:18,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:18,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ac451152eba8416fbca4fa3acdad98d4_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456917359/Put/seqid=0 2024-11-13T00:15:18,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742288_1464 (size=12454) 2024-11-13T00:15:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:19,032 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ac451152eba8416fbca4fa3acdad98d4_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac451152eba8416fbca4fa3acdad98d4_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:19,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bba08714f2e44579b902cacb8eb94e8, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:19,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bba08714f2e44579b902cacb8eb94e8 is 175, key is test_row_0/A:col10/1731456917359/Put/seqid=0 2024-11-13T00:15:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742289_1465 (size=31255) 2024-11-13T00:15:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:19,439 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bba08714f2e44579b902cacb8eb94e8 2024-11-13T00:15:19,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/00dc312065cb4d62a93a930a0cd6283e is 50, key is test_row_0/B:col10/1731456917359/Put/seqid=0 2024-11-13T00:15:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742290_1466 (size=12301) 2024-11-13T00:15:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:19,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456979580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456979581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456979589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456979590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456979590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456979696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456979696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456979696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456979716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456979716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,854 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/00dc312065cb4d62a93a930a0cd6283e 2024-11-13T00:15:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/2ef0207210a54b268316e20871ee4702 is 50, key is test_row_0/C:col10/1731456917359/Put/seqid=0 2024-11-13T00:15:19,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742291_1467 (size=12301) 2024-11-13T00:15:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456979906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456979906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456979906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456979936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456979936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456980226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456980226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456980226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456980256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456980256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,265 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/2ef0207210a54b268316e20871ee4702 2024-11-13T00:15:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bba08714f2e44579b902cacb8eb94e8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8 2024-11-13T00:15:20,273 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8, entries=150, sequenceid=273, filesize=30.5 K 2024-11-13T00:15:20,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/00dc312065cb4d62a93a930a0cd6283e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/00dc312065cb4d62a93a930a0cd6283e 2024-11-13T00:15:20,277 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/00dc312065cb4d62a93a930a0cd6283e, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T00:15:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/2ef0207210a54b268316e20871ee4702 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2ef0207210a54b268316e20871ee4702 2024-11-13T00:15:20,282 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2ef0207210a54b268316e20871ee4702, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T00:15:20,282 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 96d540c45076fbf64e35c80767cd2e16 in 1664ms, sequenceid=273, compaction requested=true 2024-11-13T00:15:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T00:15:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T00:15:20,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T00:15:20,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0440 sec 2024-11-13T00:15:20,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.0490 sec 2024-11-13T00:15:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:20,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:15:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:20,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:20,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:20,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:20,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132b0a828fabea4f0fbfa6d939b141588e_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:20,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742292_1468 (size=14994) 2024-11-13T00:15:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456980766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456980766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456980766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456980774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456980775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456980886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456980886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:20,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456980896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456981096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456981096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456981116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:21,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132b0a828fabea4f0fbfa6d939b141588e_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132b0a828fabea4f0fbfa6d939b141588e_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:21,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/87d7d9c14b45460aaf6423b4929c27bd, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:21,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/87d7d9c14b45460aaf6423b4929c27bd is 175, key is test_row_0/A:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:21,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742293_1469 (size=39949) 2024-11-13T00:15:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T00:15:21,406 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:21,407 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-13T00:15:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T00:15:21,408 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:21,409 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:21,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456981406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456981406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456981435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T00:15:21,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:21,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:21,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:21,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:21,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/87d7d9c14b45460aaf6423b4929c27bd 2024-11-13T00:15:21,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/ab92de33397344cf936c55b784d77828 is 50, key is test_row_0/B:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:21,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742294_1470 (size=12301) 2024-11-13T00:15:21,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:21,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:21,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:21,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:21,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T00:15:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456981786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,796 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 96d540c45076fbf64e35c80767cd2e16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456981796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,808 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 96d540c45076fbf64e35c80767cd2e16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:21,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456981926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456981926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456981956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:21,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/ab92de33397344cf936c55b784d77828 2024-11-13T00:15:21,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/8306183e011a4c91a8b45167841f68a0 is 50, key is test_row_0/C:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:21,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742295_1471 (size=12301) 2024-11-13T00:15:22,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T00:15:22,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:22,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/8306183e011a4c91a8b45167841f68a0 2024-11-13T00:15:22,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/87d7d9c14b45460aaf6423b4929c27bd as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd 2024-11-13T00:15:22,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd, entries=200, sequenceid=293, filesize=39.0 K 2024-11-13T00:15:22,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/ab92de33397344cf936c55b784d77828 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/ab92de33397344cf936c55b784d77828 2024-11-13T00:15:22,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/ab92de33397344cf936c55b784d77828, entries=150, sequenceid=293, filesize=12.0 K 2024-11-13T00:15:22,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/8306183e011a4c91a8b45167841f68a0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8306183e011a4c91a8b45167841f68a0 2024-11-13T00:15:22,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8306183e011a4c91a8b45167841f68a0, entries=150, sequenceid=293, filesize=12.0 K 2024-11-13T00:15:22,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 96d540c45076fbf64e35c80767cd2e16 in 1661ms, sequenceid=293, compaction requested=true 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:22,410 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:22,410 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:22,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:22,411 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142688 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:22,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:22,411 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:22,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:22,411 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,411 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,411 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/78dc5d91be184a36bf6ddbc81dd293d5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d5e2fc7d4ebc4113845628fe913bb6e8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/00dc312065cb4d62a93a930a0cd6283e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/ab92de33397344cf936c55b784d77828] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=48.4 K 2024-11-13T00:15:22,411 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/0385494c9f82404e8647b6171f3f0cf0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=139.3 K 2024-11-13T00:15:22,411 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,411 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/0385494c9f82404e8647b6171f3f0cf0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd] 2024-11-13T00:15:22,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 78dc5d91be184a36bf6ddbc81dd293d5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731456915953 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0385494c9f82404e8647b6171f3f0cf0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731456915953 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e2fc7d4ebc4113845628fe913bb6e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731456916679 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d796653e7fd4c408ee8007c38c471ae, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731456916679 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 00dc312065cb4d62a93a930a0cd6283e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731456917359 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bba08714f2e44579b902cacb8eb94e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731456917359 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ab92de33397344cf936c55b784d77828, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731456919579 2024-11-13T00:15:22,412 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d7d9c14b45460aaf6423b4929c27bd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731456919579 2024-11-13T00:15:22,420 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:22,422 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:22,423 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/4f6f57dd05ab47d388c06d376caffe85 is 50, key is test_row_0/B:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:22,425 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bcd6c87f8a994656ac899abccae0a572_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:22,428 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bcd6c87f8a994656ac899abccae0a572_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:22,428 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bcd6c87f8a994656ac899abccae0a572_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:22,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742296_1472 (size=13017) 2024-11-13T00:15:22,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742297_1473 (size=4469) 2024-11-13T00:15:22,439 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/4f6f57dd05ab47d388c06d376caffe85 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4f6f57dd05ab47d388c06d376caffe85 2024-11-13T00:15:22,443 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into 4f6f57dd05ab47d388c06d376caffe85(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:22,443 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:22,443 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=12, startTime=1731456922410; duration=0sec 2024-11-13T00:15:22,443 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:22,443 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:22,443 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:22,444 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:22,444 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:22,445 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,445 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/769a5fe262b84b8ba4cbf91298cf1ca3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8232143e26ce49e8a3454f794597250a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2ef0207210a54b268316e20871ee4702, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8306183e011a4c91a8b45167841f68a0] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=48.4 K 2024-11-13T00:15:22,445 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 769a5fe262b84b8ba4cbf91298cf1ca3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731456915953 2024-11-13T00:15:22,445 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8232143e26ce49e8a3454f794597250a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731456916679 2024-11-13T00:15:22,446 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef0207210a54b268316e20871ee4702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731456917359 2024-11-13T00:15:22,446 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8306183e011a4c91a8b45167841f68a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731456919579 2024-11-13T00:15:22,452 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:22,452 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/6c6b08abace54dd78779bbad142a72df is 50, key is test_row_0/C:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:22,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742298_1474 (size=13017) 2024-11-13T00:15:22,462 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/6c6b08abace54dd78779bbad142a72df as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6c6b08abace54dd78779bbad142a72df 2024-11-13T00:15:22,465 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into 6c6b08abace54dd78779bbad142a72df(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:22,465 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:22,465 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=12, startTime=1731456922410; duration=0sec 2024-11-13T00:15:22,465 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:22,465 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:22,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:22,480 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:22,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139b77c73caf19406bb27adea7cc4e8cad_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456920774/Put/seqid=0 2024-11-13T00:15:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742299_1475 (size=12454) 2024-11-13T00:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T00:15:22,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:15:22,838 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#394 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:22,838 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/740af2a5686f4065ba015b7f8eab7a89 is 175, key is test_row_0/A:col10/1731456919579/Put/seqid=0 2024-11-13T00:15:22,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742300_1476 (size=31971) 2024-11-13T00:15:22,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:22,893 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139b77c73caf19406bb27adea7cc4e8cad_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139b77c73caf19406bb27adea7cc4e8cad_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/98d22fa938674d2f85fe611c93d206d3, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/98d22fa938674d2f85fe611c93d206d3 is 175, key is test_row_0/A:col10/1731456920774/Put/seqid=0 2024-11-13T00:15:22,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742301_1477 (size=31255) 2024-11-13T00:15:22,898 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/98d22fa938674d2f85fe611c93d206d3 2024-11-13T00:15:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/57a120ad069a44a4bcfd92117bb69a74 is 50, key is test_row_0/B:col10/1731456920774/Put/seqid=0 2024-11-13T00:15:22,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742302_1478 (size=12301) 2024-11-13T00:15:22,909 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/57a120ad069a44a4bcfd92117bb69a74 2024-11-13T00:15:22,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/28c8a6ca43454cca915c48e8a0b9b72d is 50, key is test_row_0/C:col10/1731456920774/Put/seqid=0 2024-11-13T00:15:22,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742303_1479 (size=12301) 2024-11-13T00:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456983013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456983020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456983021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456983126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456983135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456983136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,250 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/740af2a5686f4065ba015b7f8eab7a89 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/740af2a5686f4065ba015b7f8eab7a89 2024-11-13T00:15:23,254 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into 740af2a5686f4065ba015b7f8eab7a89(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:23,254 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:23,254 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=12, startTime=1731456922410; duration=0sec 2024-11-13T00:15:23,254 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:23,254 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:23,319 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/28c8a6ca43454cca915c48e8a0b9b72d 2024-11-13T00:15:23,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/98d22fa938674d2f85fe611c93d206d3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3 2024-11-13T00:15:23,327 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3, entries=150, sequenceid=307, filesize=30.5 K 2024-11-13T00:15:23,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/57a120ad069a44a4bcfd92117bb69a74 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/57a120ad069a44a4bcfd92117bb69a74 2024-11-13T00:15:23,331 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/57a120ad069a44a4bcfd92117bb69a74, entries=150, sequenceid=307, filesize=12.0 K 2024-11-13T00:15:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/28c8a6ca43454cca915c48e8a0b9b72d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/28c8a6ca43454cca915c48e8a0b9b72d 2024-11-13T00:15:23,334 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/28c8a6ca43454cca915c48e8a0b9b72d, entries=150, sequenceid=307, filesize=12.0 K 2024-11-13T00:15:23,335 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 96d540c45076fbf64e35c80767cd2e16 in 855ms, sequenceid=307, compaction requested=false 2024-11-13T00:15:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-13T00:15:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-13T00:15:23,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T00:15:23,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-13T00:15:23,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9320 sec 2024-11-13T00:15:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:23,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T00:15:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:23,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111303d5835e01834e3cbb26475bff00b4a6_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:23,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742304_1480 (size=14994) 2024-11-13T00:15:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456983372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456983373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456983376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456983486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456983486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456983496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T00:15:23,556 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:23,556 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-13T00:15:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:23,558 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:23,558 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:23,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:23,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456983696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456983696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:23,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456983706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:23,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111303d5835e01834e3cbb26475bff00b4a6_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111303d5835e01834e3cbb26475bff00b4a6_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:23,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bf060a0bf9642adbfa0688cd48db9d4, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:23,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bf060a0bf9642adbfa0688cd48db9d4 is 175, key is test_row_0/A:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:23,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742305_1481 (size=39949) 2024-11-13T00:15:23,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731456983816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,821 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 96d540c45076fbf64e35c80767cd2e16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731456983826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:23,834 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 96d540c45076fbf64e35c80767cd2e16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:23,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:23,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:23,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:23,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:23,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:23,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:24,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456984006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:24,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:24,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456984006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:24,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:24,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456984016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:24,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bf060a0bf9642adbfa0688cd48db9d4 2024-11-13T00:15:24,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/5f05507558c9456e8b22083576ee08fe is 50, key is test_row_0/B:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742306_1482 (size=12301) 2024-11-13T00:15:24,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45182 deadline: 1731456984526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45200 deadline: 1731456984526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1731456984527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:15:24,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/5f05507558c9456e8b22083576ee08fe 2024-11-13T00:15:24,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/6ff3260c8cae4798a17a29059980ea41 is 50, key is test_row_0/C:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742307_1483 (size=12301) 2024-11-13T00:15:24,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:24,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:24,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:25,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/6ff3260c8cae4798a17a29059980ea41 2024-11-13T00:15:25,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/9bf060a0bf9642adbfa0688cd48db9d4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4 2024-11-13T00:15:25,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4, entries=200, sequenceid=329, filesize=39.0 K 2024-11-13T00:15:25,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/5f05507558c9456e8b22083576ee08fe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/5f05507558c9456e8b22083576ee08fe 2024-11-13T00:15:25,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/5f05507558c9456e8b22083576ee08fe, entries=150, sequenceid=329, filesize=12.0 K 2024-11-13T00:15:25,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/6ff3260c8cae4798a17a29059980ea41 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6ff3260c8cae4798a17a29059980ea41 2024-11-13T00:15:25,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6ff3260c8cae4798a17a29059980ea41, entries=150, sequenceid=329, filesize=12.0 K 2024-11-13T00:15:25,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 96d540c45076fbf64e35c80767cd2e16 in 1662ms, sequenceid=329, compaction requested=true 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96d540c45076fbf64e35c80767cd2e16:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:25,016 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:25,016 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:25,017 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:25,017 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:25,017 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/B is initiating minor compaction (all files) 2024-11-13T00:15:25,017 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/A is initiating minor compaction (all files) 2024-11-13T00:15:25,017 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/B in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:25,017 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/A in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:25,017 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4f6f57dd05ab47d388c06d376caffe85, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/57a120ad069a44a4bcfd92117bb69a74, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/5f05507558c9456e8b22083576ee08fe] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=36.7 K 2024-11-13T00:15:25,017 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/740af2a5686f4065ba015b7f8eab7a89, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=100.8 K 2024-11-13T00:15:25,017 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:25,017 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/740af2a5686f4065ba015b7f8eab7a89, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4] 2024-11-13T00:15:25,018 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 740af2a5686f4065ba015b7f8eab7a89, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731456919579 2024-11-13T00:15:25,018 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f6f57dd05ab47d388c06d376caffe85, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731456919579 2024-11-13T00:15:25,018 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a120ad069a44a4bcfd92117bb69a74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731456920761 2024-11-13T00:15:25,018 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d22fa938674d2f85fe611c93d206d3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731456920761 2024-11-13T00:15:25,018 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bf060a0bf9642adbfa0688cd48db9d4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731456922990 2024-11-13T00:15:25,018 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f05507558c9456e8b22083576ee08fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731456922990 2024-11-13T00:15:25,023 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:25,025 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:25,025 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ab9e3356e9bc4ff7ad9f51e323c0eba0_96d540c45076fbf64e35c80767cd2e16 store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:25,025 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/67ec932f7f4b496fac410f9947fbec50 is 50, key is test_row_0/B:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:25,027 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ab9e3356e9bc4ff7ad9f51e323c0eba0_96d540c45076fbf64e35c80767cd2e16, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:25,027 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ab9e3356e9bc4ff7ad9f51e323c0eba0_96d540c45076fbf64e35c80767cd2e16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:25,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742308_1484 (size=13119) 2024-11-13T00:15:25,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742309_1485 (size=4469) 2024-11-13T00:15:25,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T00:15:25,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:25,089 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:15:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:25,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111381338f43b2ad4cadae25f61563ca350c_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456923364/Put/seqid=0 2024-11-13T00:15:25,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742310_1486 (size=12454) 2024-11-13T00:15:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:25,107 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111381338f43b2ad4cadae25f61563ca350c_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111381338f43b2ad4cadae25f61563ca350c_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:25,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/e2d54d0e26fe404f9ac9029702470bab, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:25,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/e2d54d0e26fe404f9ac9029702470bab is 175, key is test_row_0/A:col10/1731456923364/Put/seqid=0 2024-11-13T00:15:25,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742311_1487 (size=31255) 2024-11-13T00:15:25,122 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/e2d54d0e26fe404f9ac9029702470bab 2024-11-13T00:15:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/1af1a85e73e242c48ac6c3b557e60951 is 50, key is test_row_0/B:col10/1731456923364/Put/seqid=0 2024-11-13T00:15:25,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742312_1488 (size=12301) 2024-11-13T00:15:25,432 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#A#compaction#403 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:25,432 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/faa9b3db52a54290a12e2aaf2232117f is 175, key is test_row_0/A:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:25,435 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/67ec932f7f4b496fac410f9947fbec50 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/67ec932f7f4b496fac410f9947fbec50 2024-11-13T00:15:25,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742313_1489 (size=32073) 2024-11-13T00:15:25,441 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/faa9b3db52a54290a12e2aaf2232117f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/faa9b3db52a54290a12e2aaf2232117f 2024-11-13T00:15:25,447 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/A of 96d540c45076fbf64e35c80767cd2e16 into faa9b3db52a54290a12e2aaf2232117f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:25,447 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:25,448 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/A, priority=13, startTime=1731456925016; duration=0sec 2024-11-13T00:15:25,448 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:25,448 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:A 2024-11-13T00:15:25,448 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/B of 96d540c45076fbf64e35c80767cd2e16 into 67ec932f7f4b496fac410f9947fbec50(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:25,448 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:25,448 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/B, priority=13, startTime=1731456925016; duration=0sec 2024-11-13T00:15:25,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:25,449 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:25,449 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:B 2024-11-13T00:15:25,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:25,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 96d540c45076fbf64e35c80767cd2e16/C is initiating minor compaction (all files) 2024-11-13T00:15:25,449 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96d540c45076fbf64e35c80767cd2e16/C in TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:25,450 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6c6b08abace54dd78779bbad142a72df, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/28c8a6ca43454cca915c48e8a0b9b72d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6ff3260c8cae4798a17a29059980ea41] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp, totalSize=36.7 K 2024-11-13T00:15:25,450 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c6b08abace54dd78779bbad142a72df, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731456919579 2024-11-13T00:15:25,450 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c8a6ca43454cca915c48e8a0b9b72d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731456920761 2024-11-13T00:15:25,451 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff3260c8cae4798a17a29059980ea41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731456922990 2024-11-13T00:15:25,458 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96d540c45076fbf64e35c80767cd2e16#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:25,458 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/5bd1d133acb94bd19a418f795eb8da8e is 50, key is test_row_0/C:col10/1731456923011/Put/seqid=0 2024-11-13T00:15:25,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742314_1490 (size=13119) 2024-11-13T00:15:25,467 INFO [Thread-1754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1754. 2024-11-13T00:15:25,467 INFO [Thread-1752 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1752. 2024-11-13T00:15:25,467 DEBUG [Thread-1754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,467 DEBUG [Thread-1752 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,467 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,467 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,468 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,468 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,468 INFO [Thread-1748 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1748. 2024-11-13T00:15:25,468 DEBUG [Thread-1748 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,468 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,468 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,474 INFO [Thread-1750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1750. 2024-11-13T00:15:25,474 DEBUG [Thread-1750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,474 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,474 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,474 INFO [Thread-1756 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1756. 2024-11-13T00:15:25,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,474 DEBUG [Thread-1756 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,474 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,474 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,543 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/1af1a85e73e242c48ac6c3b557e60951 2024-11-13T00:15:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:25,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. as already flushing 2024-11-13T00:15:25,548 INFO [Thread-1743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1743. 2024-11-13T00:15:25,548 INFO [Thread-1745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1745. 2024-11-13T00:15:25,548 DEBUG [Thread-1743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,548 DEBUG [Thread-1745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,548 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,548 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,548 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,548 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/b8df6bad53f6436687b4e30d9f4bbd27 is 50, key is test_row_0/C:col10/1731456923364/Put/seqid=0 2024-11-13T00:15:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742315_1491 (size=12301) 2024-11-13T00:15:25,557 INFO [Thread-1739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1739. 2024-11-13T00:15:25,557 DEBUG [Thread-1739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:25,557 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,557 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:25,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:25,866 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/5bd1d133acb94bd19a418f795eb8da8e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/5bd1d133acb94bd19a418f795eb8da8e 2024-11-13T00:15:25,870 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96d540c45076fbf64e35c80767cd2e16/C of 96d540c45076fbf64e35c80767cd2e16 into 5bd1d133acb94bd19a418f795eb8da8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:25,870 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:25,870 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., storeName=96d540c45076fbf64e35c80767cd2e16/C, priority=13, startTime=1731456925016; duration=0sec 2024-11-13T00:15:25,870 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:25,870 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96d540c45076fbf64e35c80767cd2e16:C 2024-11-13T00:15:25,952 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/b8df6bad53f6436687b4e30d9f4bbd27 2024-11-13T00:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/e2d54d0e26fe404f9ac9029702470bab as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/e2d54d0e26fe404f9ac9029702470bab 2024-11-13T00:15:25,959 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/e2d54d0e26fe404f9ac9029702470bab, entries=150, sequenceid=340, filesize=30.5 K 2024-11-13T00:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/1af1a85e73e242c48ac6c3b557e60951 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/1af1a85e73e242c48ac6c3b557e60951 2024-11-13T00:15:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/1af1a85e73e242c48ac6c3b557e60951, entries=150, sequenceid=340, filesize=12.0 K 2024-11-13T00:15:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/b8df6bad53f6436687b4e30d9f4bbd27 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/b8df6bad53f6436687b4e30d9f4bbd27 2024-11-13T00:15:25,965 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/b8df6bad53f6436687b4e30d9f4bbd27, entries=150, sequenceid=340, filesize=12.0 K 2024-11-13T00:15:25,966 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 96d540c45076fbf64e35c80767cd2e16 in 877ms, sequenceid=340, compaction requested=false 2024-11-13T00:15:25,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 96d540c45076fbf64e35c80767cd2e16: 2024-11-13T00:15:25,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:25,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-13T00:15:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-13T00:15:25,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T00:15:25,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4080 sec 2024-11-13T00:15:25,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.4120 sec 2024-11-13T00:15:26,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6362cce124054c0b924ca055e4416874, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/1ea98089b44d49ba9f816b93e430970d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/238b0ec0c47e48968aeb7848ebf41de6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/47b06864cfa747ea819316ad19d1c636, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/0385494c9f82404e8647b6171f3f0cf0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/740af2a5686f4065ba015b7f8eab7a89, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4] to archive 2024-11-13T00:15:26,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:26,980 WARN [regionserver/cb28580903fb:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-13T00:15:26,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6df43ce7d604491096520075d316eb82 2024-11-13T00:15:26,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9b6eb547d24249848c1f9cce2a45395f 2024-11-13T00:15:26,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6362cce124054c0b924ca055e4416874 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6362cce124054c0b924ca055e4416874 2024-11-13T00:15:26,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/39e2cd14902f4778aab0ab6128df97ce 2024-11-13T00:15:26,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/bab9da4df2d240ac8bc592970cd5ff3d 2024-11-13T00:15:26,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/1ea98089b44d49ba9f816b93e430970d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/1ea98089b44d49ba9f816b93e430970d 2024-11-13T00:15:26,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a4fed389c6364b57b44347d607c506db 2024-11-13T00:15:26,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/df8c11e85c1946a7a6879da57c0baaec 2024-11-13T00:15:26,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/da0249572e0740578c71ac57bb27eb7f 2024-11-13T00:15:26,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/a7f2118cba85438b851cd5b455c35f91 2024-11-13T00:15:26,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/238b0ec0c47e48968aeb7848ebf41de6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/238b0ec0c47e48968aeb7848ebf41de6 2024-11-13T00:15:26,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/c1cc9c0342b642aba171f9d223052b3e 2024-11-13T00:15:26,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/f12386b923c942d5ba9dafc8e5c91759 2024-11-13T00:15:26,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/51dc3c00cbf14fc2b27030c30f850430 2024-11-13T00:15:26,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/47b06864cfa747ea819316ad19d1c636 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/47b06864cfa747ea819316ad19d1c636 2024-11-13T00:15:26,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/edfbcef2a12d4ede946cee323eb38df8 2024-11-13T00:15:26,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/0385494c9f82404e8647b6171f3f0cf0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/0385494c9f82404e8647b6171f3f0cf0 2024-11-13T00:15:26,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/6a76c284785543379ac3e4131d77ca81 2024-11-13T00:15:26,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/4d796653e7fd4c408ee8007c38c471ae 2024-11-13T00:15:26,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bba08714f2e44579b902cacb8eb94e8 2024-11-13T00:15:26,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/87d7d9c14b45460aaf6423b4929c27bd 2024-11-13T00:15:26,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/740af2a5686f4065ba015b7f8eab7a89 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/740af2a5686f4065ba015b7f8eab7a89 2024-11-13T00:15:26,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/98d22fa938674d2f85fe611c93d206d3 2024-11-13T00:15:26,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/9bf060a0bf9642adbfa0688cd48db9d4 2024-11-13T00:15:27,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/34307627197e4bd5b83aea31b0a11726, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4e6568e0bcde4263a481c7e6ad5e5342, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9bdfbe23f3614e2c93f6568f7a803f01, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/70a156cc4c284f7fbd65e7eb55b62dd7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/248b93fa79e5416883e0a5d1917a6782, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/56e562148cf04b129553048b882694f7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/aa157e1c3eea4aac827e3566ea765f22, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3299379720c54dfe93f8f8811d9a8314, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/a93c37da76644892b92e62c089daf4b8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3d019019579a43848706f05070647f7b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/be33284d88fb4e8b91aa5c2a8d4670a1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/0bb57e0fbced48688a0d2b5bba94568b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9b240db90c43407296e80ab01ec69e05, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d69b571110094646844ee7651c876084, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/7a964ec04d5f47ef93634f347eb4e802, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d71f5c91111c473dabbb241662bb4139, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/78dc5d91be184a36bf6ddbc81dd293d5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/2a9318bd9967436098cee8ff05f27fa7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d5e2fc7d4ebc4113845628fe913bb6e8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/00dc312065cb4d62a93a930a0cd6283e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4f6f57dd05ab47d388c06d376caffe85, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/ab92de33397344cf936c55b784d77828, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/57a120ad069a44a4bcfd92117bb69a74, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/5f05507558c9456e8b22083576ee08fe] to archive 2024-11-13T00:15:27,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:27,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/34307627197e4bd5b83aea31b0a11726 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/34307627197e4bd5b83aea31b0a11726 2024-11-13T00:15:27,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4e6568e0bcde4263a481c7e6ad5e5342 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4e6568e0bcde4263a481c7e6ad5e5342 2024-11-13T00:15:27,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9bdfbe23f3614e2c93f6568f7a803f01 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9bdfbe23f3614e2c93f6568f7a803f01 2024-11-13T00:15:27,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/70a156cc4c284f7fbd65e7eb55b62dd7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/70a156cc4c284f7fbd65e7eb55b62dd7 2024-11-13T00:15:27,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/248b93fa79e5416883e0a5d1917a6782 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/248b93fa79e5416883e0a5d1917a6782 2024-11-13T00:15:27,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/56e562148cf04b129553048b882694f7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/56e562148cf04b129553048b882694f7 2024-11-13T00:15:27,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/aa157e1c3eea4aac827e3566ea765f22 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/aa157e1c3eea4aac827e3566ea765f22 2024-11-13T00:15:27,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3299379720c54dfe93f8f8811d9a8314 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3299379720c54dfe93f8f8811d9a8314 2024-11-13T00:15:27,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/a93c37da76644892b92e62c089daf4b8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/a93c37da76644892b92e62c089daf4b8 2024-11-13T00:15:27,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3d019019579a43848706f05070647f7b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/3d019019579a43848706f05070647f7b 2024-11-13T00:15:27,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/be33284d88fb4e8b91aa5c2a8d4670a1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/be33284d88fb4e8b91aa5c2a8d4670a1 2024-11-13T00:15:27,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/0bb57e0fbced48688a0d2b5bba94568b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/0bb57e0fbced48688a0d2b5bba94568b 2024-11-13T00:15:27,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9b240db90c43407296e80ab01ec69e05 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/9b240db90c43407296e80ab01ec69e05 2024-11-13T00:15:27,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d69b571110094646844ee7651c876084 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d69b571110094646844ee7651c876084 2024-11-13T00:15:27,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/7a964ec04d5f47ef93634f347eb4e802 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/7a964ec04d5f47ef93634f347eb4e802 2024-11-13T00:15:27,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d71f5c91111c473dabbb241662bb4139 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d71f5c91111c473dabbb241662bb4139 2024-11-13T00:15:27,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/78dc5d91be184a36bf6ddbc81dd293d5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/78dc5d91be184a36bf6ddbc81dd293d5 2024-11-13T00:15:27,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/2a9318bd9967436098cee8ff05f27fa7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/2a9318bd9967436098cee8ff05f27fa7 2024-11-13T00:15:27,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d5e2fc7d4ebc4113845628fe913bb6e8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d5e2fc7d4ebc4113845628fe913bb6e8 2024-11-13T00:15:27,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/00dc312065cb4d62a93a930a0cd6283e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/00dc312065cb4d62a93a930a0cd6283e 2024-11-13T00:15:27,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4f6f57dd05ab47d388c06d376caffe85 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/4f6f57dd05ab47d388c06d376caffe85 2024-11-13T00:15:27,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/ab92de33397344cf936c55b784d77828 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/ab92de33397344cf936c55b784d77828 2024-11-13T00:15:27,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/57a120ad069a44a4bcfd92117bb69a74 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/57a120ad069a44a4bcfd92117bb69a74 2024-11-13T00:15:27,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/5f05507558c9456e8b22083576ee08fe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/5f05507558c9456e8b22083576ee08fe 2024-11-13T00:15:27,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a7cadaee62174aea9405501781eaff0b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17d1d2a377a9417bb4edd1708e2fdb42, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/fb966f04ec2645349c9d1eb61b60a1d4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/cf90babcac564851b3dcac0f936a1eda, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/3f121e4f96e84b02afcc3ba30a87447a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/0d7349de4e1e4ddcaa0720329b979ed8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/d2f59b187917439a8bcf21318adc5b9c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/38e3c6582ba44ea48f5d46aa2174e59a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17de4788f1a746729d8b1acb54fb7b66, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/994468fb14c54ef9bd14ebcabba1f058, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1766d2e8cce7461d8dcb11e1fab3b663, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/150a3727d0c047b6950a583edc19ace7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a66a0854e118442c87078d7996a3de0e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/808e1239118e4f61b016265744e78e33, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/9a5eb419c5d4482882ff3c6a29f8eac0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1b9dfb8826984804b2f5c317e0a0f919, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/769a5fe262b84b8ba4cbf91298cf1ca3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/911e707f75934d61b555fd80337eeeeb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8232143e26ce49e8a3454f794597250a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2ef0207210a54b268316e20871ee4702, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6c6b08abace54dd78779bbad142a72df, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8306183e011a4c91a8b45167841f68a0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/28c8a6ca43454cca915c48e8a0b9b72d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6ff3260c8cae4798a17a29059980ea41] to archive 2024-11-13T00:15:27,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:27,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a7cadaee62174aea9405501781eaff0b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a7cadaee62174aea9405501781eaff0b 2024-11-13T00:15:27,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17d1d2a377a9417bb4edd1708e2fdb42 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17d1d2a377a9417bb4edd1708e2fdb42 2024-11-13T00:15:27,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/fb966f04ec2645349c9d1eb61b60a1d4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/fb966f04ec2645349c9d1eb61b60a1d4 2024-11-13T00:15:27,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/cf90babcac564851b3dcac0f936a1eda to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/cf90babcac564851b3dcac0f936a1eda 2024-11-13T00:15:27,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/3f121e4f96e84b02afcc3ba30a87447a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/3f121e4f96e84b02afcc3ba30a87447a 2024-11-13T00:15:27,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/0d7349de4e1e4ddcaa0720329b979ed8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/0d7349de4e1e4ddcaa0720329b979ed8 2024-11-13T00:15:27,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/d2f59b187917439a8bcf21318adc5b9c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/d2f59b187917439a8bcf21318adc5b9c 2024-11-13T00:15:27,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/38e3c6582ba44ea48f5d46aa2174e59a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/38e3c6582ba44ea48f5d46aa2174e59a 2024-11-13T00:15:27,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17de4788f1a746729d8b1acb54fb7b66 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/17de4788f1a746729d8b1acb54fb7b66 2024-11-13T00:15:27,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/994468fb14c54ef9bd14ebcabba1f058 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/994468fb14c54ef9bd14ebcabba1f058 2024-11-13T00:15:27,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1766d2e8cce7461d8dcb11e1fab3b663 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1766d2e8cce7461d8dcb11e1fab3b663 2024-11-13T00:15:27,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/150a3727d0c047b6950a583edc19ace7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/150a3727d0c047b6950a583edc19ace7 2024-11-13T00:15:27,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a66a0854e118442c87078d7996a3de0e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/a66a0854e118442c87078d7996a3de0e 2024-11-13T00:15:27,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/808e1239118e4f61b016265744e78e33 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/808e1239118e4f61b016265744e78e33 2024-11-13T00:15:27,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/9a5eb419c5d4482882ff3c6a29f8eac0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/9a5eb419c5d4482882ff3c6a29f8eac0 2024-11-13T00:15:27,038 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1b9dfb8826984804b2f5c317e0a0f919 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/1b9dfb8826984804b2f5c317e0a0f919 2024-11-13T00:15:27,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/769a5fe262b84b8ba4cbf91298cf1ca3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/769a5fe262b84b8ba4cbf91298cf1ca3 2024-11-13T00:15:27,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/911e707f75934d61b555fd80337eeeeb to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/911e707f75934d61b555fd80337eeeeb 2024-11-13T00:15:27,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8232143e26ce49e8a3454f794597250a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8232143e26ce49e8a3454f794597250a 2024-11-13T00:15:27,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2ef0207210a54b268316e20871ee4702 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2ef0207210a54b268316e20871ee4702 2024-11-13T00:15:27,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6c6b08abace54dd78779bbad142a72df to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6c6b08abace54dd78779bbad142a72df 2024-11-13T00:15:27,043 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8306183e011a4c91a8b45167841f68a0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/8306183e011a4c91a8b45167841f68a0 2024-11-13T00:15:27,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/28c8a6ca43454cca915c48e8a0b9b72d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/28c8a6ca43454cca915c48e8a0b9b72d 2024-11-13T00:15:27,045 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6ff3260c8cae4798a17a29059980ea41 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/6ff3260c8cae4798a17a29059980ea41 2024-11-13T00:15:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T00:15:27,716 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:27,848 INFO [Thread-1741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1741. 2024-11-13T00:15:27,848 INFO [Thread-1737 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1737. 2024-11-13T00:15:27,848 DEBUG [Thread-1737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:27,848 DEBUG [Thread-1741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:27,848 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:27,848 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:27,848 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:27,848 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:27,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:27,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:27,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T00:15:27,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-13T00:15:27,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-13T00:15:27,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1098 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3292 rows 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1105 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3314 rows 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1103 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-13T00:15:27,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3308 rows 2024-11-13T00:15:27,849 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:15:27,849 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:15:27,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:27,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:27,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:27,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T00:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T00:15:27,853 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456927852"}]},"ts":"1731456927852"} 2024-11-13T00:15:27,854 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T00:15:27,854 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T00:15:27,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:15:27,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, UNASSIGN}] 2024-11-13T00:15:27,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, UNASSIGN 2024-11-13T00:15:27,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:27,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, UNASSIGN because future has completed 2024-11-13T00:15:27,858 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:15:27,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T00:15:28,010 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 96d540c45076fbf64e35c80767cd2e16, disabling compactions & flushes 2024-11-13T00:15:28,010 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. after waiting 0 ms 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:28,010 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 96d540c45076fbf64e35c80767cd2e16 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=A 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=B 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:28,010 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96d540c45076fbf64e35c80767cd2e16, store=C 2024-11-13T00:15:28,011 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:28,015 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113968d2fd22f304083b1513e41460c4f7b_96d540c45076fbf64e35c80767cd2e16 is 50, key is test_row_0/A:col10/1731456927846/Put/seqid=0 2024-11-13T00:15:28,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742316_1492 (size=12454) 2024-11-13T00:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T00:15:28,419 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:28,422 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113968d2fd22f304083b1513e41460c4f7b_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113968d2fd22f304083b1513e41460c4f7b_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:28,423 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/2f73ea790aef4819b719c139572d576a, store: [table=TestAcidGuarantees family=A region=96d540c45076fbf64e35c80767cd2e16] 2024-11-13T00:15:28,423 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/2f73ea790aef4819b719c139572d576a is 175, key is test_row_0/A:col10/1731456927846/Put/seqid=0 2024-11-13T00:15:28,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742317_1493 (size=31255) 2024-11-13T00:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T00:15:28,827 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/2f73ea790aef4819b719c139572d576a 2024-11-13T00:15:28,832 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d13d60f0526e466b9a1ed6647ea36201 is 50, key is test_row_0/B:col10/1731456927846/Put/seqid=0 2024-11-13T00:15:28,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742318_1494 (size=12301) 2024-11-13T00:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T00:15:29,236 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d13d60f0526e466b9a1ed6647ea36201 2024-11-13T00:15:29,241 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/2015a0189b7a42778c66ffa8e6dcec2c is 50, key is test_row_0/C:col10/1731456927846/Put/seqid=0 2024-11-13T00:15:29,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742319_1495 (size=12301) 2024-11-13T00:15:29,645 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/2015a0189b7a42778c66ffa8e6dcec2c 2024-11-13T00:15:29,648 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/A/2f73ea790aef4819b719c139572d576a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/2f73ea790aef4819b719c139572d576a 2024-11-13T00:15:29,651 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/2f73ea790aef4819b719c139572d576a, entries=150, sequenceid=351, filesize=30.5 K 2024-11-13T00:15:29,652 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/B/d13d60f0526e466b9a1ed6647ea36201 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d13d60f0526e466b9a1ed6647ea36201 2024-11-13T00:15:29,654 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d13d60f0526e466b9a1ed6647ea36201, entries=150, sequenceid=351, filesize=12.0 K 2024-11-13T00:15:29,654 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/.tmp/C/2015a0189b7a42778c66ffa8e6dcec2c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2015a0189b7a42778c66ffa8e6dcec2c 2024-11-13T00:15:29,657 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2015a0189b7a42778c66ffa8e6dcec2c, entries=150, sequenceid=351, filesize=12.0 K 2024-11-13T00:15:29,658 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 96d540c45076fbf64e35c80767cd2e16 in 1648ms, sequenceid=351, compaction requested=true 2024-11-13T00:15:29,661 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=4 2024-11-13T00:15:29,662 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. 2024-11-13T00:15:29,662 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 96d540c45076fbf64e35c80767cd2e16: Waiting for close lock at 1731456928010Running coprocessor pre-close hooks at 1731456928010Disabling compacts and flushes for region at 1731456928010Disabling writes for close at 1731456928010Obtaining lock to block concurrent updates at 1731456928010Preparing flush snapshotting stores in 96d540c45076fbf64e35c80767cd2e16 at 1731456928010Finished memstore snapshotting TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731456928011 (+1 ms)Flushing stores of TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16. at 1731456928011Flushing 96d540c45076fbf64e35c80767cd2e16/A: creating writer at 1731456928011Flushing mob file 96d540c45076fbf64e35c80767cd2e16/A: appending metadata at 1731456928015 (+4 ms)Flushing mob file 96d540c45076fbf64e35c80767cd2e16/A: closing flushed file at 1731456928015Flushing 96d540c45076fbf64e35c80767cd2e16/A: appending metadata at 1731456928423 (+408 ms)Flushing 96d540c45076fbf64e35c80767cd2e16/A: closing flushed file at 1731456928423Flushing 96d540c45076fbf64e35c80767cd2e16/B: creating writer at 1731456928830 (+407 ms)Flushing 96d540c45076fbf64e35c80767cd2e16/B: appending metadata at 1731456928832 (+2 ms)Flushing 96d540c45076fbf64e35c80767cd2e16/B: closing flushed file at 1731456928832Flushing 96d540c45076fbf64e35c80767cd2e16/C: creating writer at 1731456929239 (+407 ms)Flushing 96d540c45076fbf64e35c80767cd2e16/C: appending metadata at 1731456929241 (+2 ms)Flushing 96d540c45076fbf64e35c80767cd2e16/C: closing flushed file at 1731456929241Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36ae8f69: reopening flushed file at 1731456929648 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@154614a6: reopening flushed file at 1731456929651 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29579806: reopening flushed file at 1731456929654 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 96d540c45076fbf64e35c80767cd2e16 in 1648ms, sequenceid=351, compaction requested=true at 1731456929658 (+4 ms)Writing region close event to WAL at 1731456929659 (+1 ms)Running coprocessor post-close hooks at 1731456929662 (+3 ms)Closed at 1731456929662 2024-11-13T00:15:29,664 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:29,665 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=96d540c45076fbf64e35c80767cd2e16, regionState=CLOSED 2024-11-13T00:15:29,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:29,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T00:15:29,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 96d540c45076fbf64e35c80767cd2e16, server=cb28580903fb,45273,1731456804594 in 1.8090 sec 2024-11-13T00:15:29,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-13T00:15:29,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96d540c45076fbf64e35c80767cd2e16, UNASSIGN in 1.8130 sec 2024-11-13T00:15:29,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T00:15:29,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8160 sec 2024-11-13T00:15:29,673 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456929673"}]},"ts":"1731456929673"} 2024-11-13T00:15:29,674 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T00:15:29,674 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T00:15:29,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8240 sec 2024-11-13T00:15:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T00:15:29,996 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:29,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T00:15:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:29,998 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T00:15:29,999 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:30,000 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/recovered.edits] 2024-11-13T00:15:30,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/2f73ea790aef4819b719c139572d576a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/2f73ea790aef4819b719c139572d576a 2024-11-13T00:15:30,005 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/e2d54d0e26fe404f9ac9029702470bab to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/e2d54d0e26fe404f9ac9029702470bab 2024-11-13T00:15:30,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/faa9b3db52a54290a12e2aaf2232117f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/A/faa9b3db52a54290a12e2aaf2232117f 2024-11-13T00:15:30,008 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/1af1a85e73e242c48ac6c3b557e60951 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/1af1a85e73e242c48ac6c3b557e60951 2024-11-13T00:15:30,009 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/67ec932f7f4b496fac410f9947fbec50 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/67ec932f7f4b496fac410f9947fbec50 2024-11-13T00:15:30,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d13d60f0526e466b9a1ed6647ea36201 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/B/d13d60f0526e466b9a1ed6647ea36201 2024-11-13T00:15:30,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2015a0189b7a42778c66ffa8e6dcec2c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/2015a0189b7a42778c66ffa8e6dcec2c 2024-11-13T00:15:30,012 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/5bd1d133acb94bd19a418f795eb8da8e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/5bd1d133acb94bd19a418f795eb8da8e 2024-11-13T00:15:30,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/b8df6bad53f6436687b4e30d9f4bbd27 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/C/b8df6bad53f6436687b4e30d9f4bbd27 2024-11-13T00:15:30,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/recovered.edits/354.seqid to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16/recovered.edits/354.seqid 2024-11-13T00:15:30,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,016 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T00:15:30,016 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T00:15:30,017 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T00:15:30,019 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111303d5835e01834e3cbb26475bff00b4a6_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111303d5835e01834e3cbb26475bff00b4a6_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,020 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111313ab1a54b44c40f2954816160b0c87e6_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111313ab1a54b44c40f2954816160b0c87e6_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,021 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113144f5b1b2615480bb7a53e0a8d1f9ce6_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113144f5b1b2615480bb7a53e0a8d1f9ce6_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,022 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111325393768d19a4fe699fd656db9865c6c_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111325393768d19a4fe699fd656db9865c6c_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,022 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132b0a828fabea4f0fbfa6d939b141588e_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132b0a828fabea4f0fbfa6d939b141588e_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,023 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135336ae4a198844e0a773b1476ca87d89_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135336ae4a198844e0a773b1476ca87d89_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,024 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111365a74b0be08b41b4a330a4e48ef0aadc_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111365a74b0be08b41b4a330a4e48ef0aadc_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,025 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136732ca455ec3427fb13354581143ac4a_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136732ca455ec3427fb13354581143ac4a_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,026 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371f046fd225a4231a69100a7d9cbea9d_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371f046fd225a4231a69100a7d9cbea9d_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,027 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137342276210da435ca32978127598717a_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137342276210da435ca32978127598717a_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,027 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137439ddaf30eb465c837fbb08e1ed9f91_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137439ddaf30eb465c837fbb08e1ed9f91_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,028 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113753105ffcadf44dd858589120349d57c_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113753105ffcadf44dd858589120349d57c_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,029 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111381338f43b2ad4cadae25f61563ca350c_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111381338f43b2ad4cadae25f61563ca350c_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,030 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138b0554a842f7416bb7f87a9758ed2316_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138b0554a842f7416bb7f87a9758ed2316_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,030 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113946e0dd6723d42dfa3d543f813935af8_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113946e0dd6723d42dfa3d543f813935af8_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,031 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113968d2fd22f304083b1513e41460c4f7b_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113968d2fd22f304083b1513e41460c4f7b_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,032 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139b77c73caf19406bb27adea7cc4e8cad_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139b77c73caf19406bb27adea7cc4e8cad_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,033 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac451152eba8416fbca4fa3acdad98d4_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ac451152eba8416fbca4fa3acdad98d4_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,033 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b53a3589b23b47ce80382b5048ab1834_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b53a3589b23b47ce80382b5048ab1834_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,034 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d825f5f6ac094a19b4c6f1a35bf60ab1_96d540c45076fbf64e35c80767cd2e16 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d825f5f6ac094a19b4c6f1a35bf60ab1_96d540c45076fbf64e35c80767cd2e16 2024-11-13T00:15:30,034 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T00:15:30,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:30,038 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T00:15:30,040 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T00:15:30,040 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:30,040 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T00:15:30,041 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731456930041"}]},"ts":"9223372036854775807"} 2024-11-13T00:15:30,042 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T00:15:30,042 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 96d540c45076fbf64e35c80767cd2e16, NAME => 'TestAcidGuarantees,,1731456902356.96d540c45076fbf64e35c80767cd2e16.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T00:15:30,042 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T00:15:30,042 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731456930042"}]},"ts":"9223372036854775807"} 2024-11-13T00:15:30,044 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T00:15:30,044 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:30,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-13T00:15:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T00:15:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T00:15:30,106 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:30,115 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 252), OpenFileDescriptor=457 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 442), ProcessCount=11 (was 11), AvailableMemoryMB=844 (was 932) 2024-11-13T00:15:30,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=844 2024-11-13T00:15:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:15:30,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:15:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:30,127 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:15:30,127 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:30,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-13T00:15:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-13T00:15:30,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:15:30,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742320_1496 (size=1061) 2024-11-13T00:15:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-13T00:15:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-13T00:15:30,534 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:15:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742321_1497 (size=53) 2024-11-13T00:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-13T00:15:30,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:15:30,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a66cd1bb022d966edc952b066b868a43, disabling compactions & flushes 2024-11-13T00:15:30,939 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:30,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:30,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. after waiting 0 ms 2024-11-13T00:15:30,939 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:30,939 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:30,940 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a66cd1bb022d966edc952b066b868a43: Waiting for close lock at 1731456930939Disabling compacts and flushes for region at 1731456930939Disabling writes for close at 1731456930939Writing region close event to WAL at 1731456930939Closed at 1731456930939 2024-11-13T00:15:30,941 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:15:30,941 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731456930941"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731456930941"}]},"ts":"1731456930941"} 2024-11-13T00:15:30,942 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:15:30,943 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:15:30,943 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456930943"}]},"ts":"1731456930943"} 2024-11-13T00:15:30,944 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T00:15:30,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, ASSIGN}] 2024-11-13T00:15:30,945 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, ASSIGN 2024-11-13T00:15:30,945 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, ASSIGN; state=OFFLINE, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=false 2024-11-13T00:15:31,096 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=a66cd1bb022d966edc952b066b868a43, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:31,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, ASSIGN because future has completed 2024-11-13T00:15:31,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:31,251 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:31,252 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:15:31,252 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,252 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:15:31,252 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,252 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,253 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,255 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:31,255 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a66cd1bb022d966edc952b066b868a43 columnFamilyName A 2024-11-13T00:15:31,255 DEBUG [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:31,255 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.HStore(327): Store=a66cd1bb022d966edc952b066b868a43/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:31,256 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,256 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:31,257 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a66cd1bb022d966edc952b066b868a43 columnFamilyName B 2024-11-13T00:15:31,257 DEBUG [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:31,257 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.HStore(327): Store=a66cd1bb022d966edc952b066b868a43/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:31,257 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,258 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:15:31,258 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a66cd1bb022d966edc952b066b868a43 columnFamilyName C 2024-11-13T00:15:31,258 DEBUG [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:31,258 INFO [StoreOpener-a66cd1bb022d966edc952b066b868a43-1 {}] regionserver.HStore(327): Store=a66cd1bb022d966edc952b066b868a43/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:15:31,258 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:31,258 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,259 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,259 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,260 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,260 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,260 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:15:31,261 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,262 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:15:31,263 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened a66cd1bb022d966edc952b066b868a43; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63804545, jitterRate=-0.049238190054893494}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:15:31,263 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:31,263 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for a66cd1bb022d966edc952b066b868a43: Running coprocessor pre-open hook at 1731456931252Writing region info on filesystem at 1731456931252Initializing all the Stores at 1731456931253 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456931253Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456931253Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456931253Cleaning up temporary data from old regions at 1731456931260 (+7 ms)Running coprocessor post-open hooks at 1731456931263 (+3 ms)Region opened successfully at 1731456931263 2024-11-13T00:15:31,264 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., pid=122, masterSystemTime=1731456931249 2024-11-13T00:15:31,265 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:31,265 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-13T00:15:31,266 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=a66cd1bb022d966edc952b066b868a43, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:31,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:31,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T00:15:31,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 in 170 msec 2024-11-13T00:15:31,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-13T00:15:31,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, ASSIGN in 325 msec 2024-11-13T00:15:31,271 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:15:31,271 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456931271"}]},"ts":"1731456931271"} 2024-11-13T00:15:31,273 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T00:15:31,273 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:15:31,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-13T00:15:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-13T00:15:32,277 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:32,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,279 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c998ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,281 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b256d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:15:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:15:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f44c1bc 2024-11-13T00:15:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:15:32,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:15:32,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4b9cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14bca6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40bf866a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,291 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7a052a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f24bba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,293 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c01f901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a3c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c063ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595a0253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2208677f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69667ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4234e233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,303 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546e0fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f58ad19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14277c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,307 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f8dcd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,308 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a250d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c43d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:15:32,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:15:32,311 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:15:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:15:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:15:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4935683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:15:32,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:15:32,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:32,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:15:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a735ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:15:32,313 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,313 DEBUG [Thread-2222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,313 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,313 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:32,314 DEBUG [Thread-2211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,314 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,314 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T00:15:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,315 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,315 DEBUG [Thread-2224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,317 DEBUG [Thread-2213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,317 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:15:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:15:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:15:32,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:15:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,324 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2] 2024-11-13T00:15:32,326 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:32,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:32,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:32,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/72c57024559f450fa31afd2fb38d202a is 50, key is test_row_0/A:col10/1731456932325/Put/seqid=0 2024-11-13T00:15:32,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742322_1498 (size=12001) 2024-11-13T00:15:32,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456932347, max timestamp=1731456932367Num uniques 859; 2024-11-13T00:15:32,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456932347, max timestamp=1731456932367Num uniques 859; 2024-11-13T00:15:32,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456932347, max timestamp=1731456932367Num uniques 859; 2024-11-13T00:15:32,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:32,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:32,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456992395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456992399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456992400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456992406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456992409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:32,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456992516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456992516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456992516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456992516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456992526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456992736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456992736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456992736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456992736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456992745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:32,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/72c57024559f450fa31afd2fb38d202a 2024-11-13T00:15:32,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/a520a7f733774332ad14545df83a7d7a is 50, key is test_row_0/B:col10/1731456932325/Put/seqid=0 2024-11-13T00:15:32,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742323_1499 (size=12001) 2024-11-13T00:15:32,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:32,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:32,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:32,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:33,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456993056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456993056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456993056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456993056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456993056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:33,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:33,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/a520a7f733774332ad14545df83a7d7a 2024-11-13T00:15:33,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/f5ad3a3f34414cfabbe8005ebce16137 is 50, key is test_row_0/C:col10/1731456932325/Put/seqid=0 2024-11-13T00:15:33,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742324_1500 (size=12001) 2024-11-13T00:15:33,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:33,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:33,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456993576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456993576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456993576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456993576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456993577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:33,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:33,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/f5ad3a3f34414cfabbe8005ebce16137 2024-11-13T00:15:33,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/72c57024559f450fa31afd2fb38d202a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/72c57024559f450fa31afd2fb38d202a 2024-11-13T00:15:33,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/72c57024559f450fa31afd2fb38d202a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T00:15:33,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/a520a7f733774332ad14545df83a7d7a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a520a7f733774332ad14545df83a7d7a 2024-11-13T00:15:33,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a520a7f733774332ad14545df83a7d7a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T00:15:33,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/f5ad3a3f34414cfabbe8005ebce16137 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f5ad3a3f34414cfabbe8005ebce16137 2024-11-13T00:15:33,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f5ad3a3f34414cfabbe8005ebce16137, entries=150, sequenceid=11, filesize=11.7 K 2024-11-13T00:15:33,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a66cd1bb022d966edc952b066b868a43 in 1347ms, sequenceid=11, compaction requested=false 2024-11-13T00:15:33,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:33,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T00:15:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:33,706 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-13T00:15:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:33,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:33,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/fa98abe6d43b4d5e9e16a327f7e8572e is 50, key is test_row_0/A:col10/1731456932384/Put/seqid=0 2024-11-13T00:15:33,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742325_1501 (size=12001) 2024-11-13T00:15:34,115 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/fa98abe6d43b4d5e9e16a327f7e8572e 2024-11-13T00:15:34,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f9ec44bdb2f34343b4a29ac371c30686 is 50, key is test_row_0/B:col10/1731456932384/Put/seqid=0 2024-11-13T00:15:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742326_1502 (size=12001) 2024-11-13T00:15:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:34,534 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f9ec44bdb2f34343b4a29ac371c30686 2024-11-13T00:15:34,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/325d13bb83754e5caa7a94642fcc06ab is 50, key is test_row_0/C:col10/1731456932384/Put/seqid=0 2024-11-13T00:15:34,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742327_1503 (size=12001) 2024-11-13T00:15:34,547 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/325d13bb83754e5caa7a94642fcc06ab 2024-11-13T00:15:34,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/fa98abe6d43b4d5e9e16a327f7e8572e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fa98abe6d43b4d5e9e16a327f7e8572e 2024-11-13T00:15:34,553 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fa98abe6d43b4d5e9e16a327f7e8572e, entries=150, sequenceid=46, filesize=11.7 K 2024-11-13T00:15:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f9ec44bdb2f34343b4a29ac371c30686 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f9ec44bdb2f34343b4a29ac371c30686 2024-11-13T00:15:34,557 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f9ec44bdb2f34343b4a29ac371c30686, entries=150, sequenceid=46, filesize=11.7 K 2024-11-13T00:15:34,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/325d13bb83754e5caa7a94642fcc06ab as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/325d13bb83754e5caa7a94642fcc06ab 2024-11-13T00:15:34,561 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/325d13bb83754e5caa7a94642fcc06ab, entries=150, sequenceid=46, filesize=11.7 K 2024-11-13T00:15:34,562 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=0 B/0 for a66cd1bb022d966edc952b066b868a43 in 855ms, sequenceid=46, compaction requested=false 2024-11-13T00:15:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T00:15:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T00:15:34,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T00:15:34,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-13T00:15:34,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2520 sec 2024-11-13T00:15:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:34,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:34,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/0622314387c74ccdabd8ab8f8f15ad21 is 50, key is test_row_0/A:col10/1731456934604/Put/seqid=0 2024-11-13T00:15:34,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742328_1504 (size=14341) 2024-11-13T00:15:34,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/0622314387c74ccdabd8ab8f8f15ad21 2024-11-13T00:15:34,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/d69b7f91b7ed45a58d90655adf9b0412 is 50, key is test_row_0/B:col10/1731456934604/Put/seqid=0 2024-11-13T00:15:34,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742329_1505 (size=12001) 2024-11-13T00:15:34,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456934624, max timestamp=1731456934641Num uniques 859; 2024-11-13T00:15:34,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456934624, max timestamp=1731456934641Num uniques 859; 2024-11-13T00:15:34,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:34,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731456934624, max timestamp=1731456934641Num uniques 859; 2024-11-13T00:15:34,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:34,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T00:15:34,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456994672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456994674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456994687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456994689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456994689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456994796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456994796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456994796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456994816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456994816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456995006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456995006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456995006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/d69b7f91b7ed45a58d90655adf9b0412 2024-11-13T00:15:35,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/728cae57a56b4b749e6b904554034063 is 50, key is test_row_0/C:col10/1731456934604/Put/seqid=0 2024-11-13T00:15:35,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742330_1506 (size=12001) 2024-11-13T00:15:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456995036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456995036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,185 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:15:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456995325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456995325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456995325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456995356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456995356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/728cae57a56b4b749e6b904554034063 2024-11-13T00:15:35,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/0622314387c74ccdabd8ab8f8f15ad21 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0622314387c74ccdabd8ab8f8f15ad21 2024-11-13T00:15:35,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0622314387c74ccdabd8ab8f8f15ad21, entries=200, sequenceid=57, filesize=14.0 K 2024-11-13T00:15:35,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/d69b7f91b7ed45a58d90655adf9b0412 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/d69b7f91b7ed45a58d90655adf9b0412 2024-11-13T00:15:35,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/d69b7f91b7ed45a58d90655adf9b0412, entries=150, sequenceid=57, filesize=11.7 K 2024-11-13T00:15:35,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/728cae57a56b4b749e6b904554034063 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/728cae57a56b4b749e6b904554034063 2024-11-13T00:15:35,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/728cae57a56b4b749e6b904554034063, entries=150, sequenceid=57, filesize=11.7 K 2024-11-13T00:15:35,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for a66cd1bb022d966edc952b066b868a43 in 859ms, sequenceid=57, compaction requested=true 2024-11-13T00:15:35,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:35,464 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:35,464 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:35,464 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:35,464 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:35,465 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:35,465 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:35,465 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:35,465 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:35,465 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/72c57024559f450fa31afd2fb38d202a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fa98abe6d43b4d5e9e16a327f7e8572e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0622314387c74ccdabd8ab8f8f15ad21] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=37.4 K 2024-11-13T00:15:35,465 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a520a7f733774332ad14545df83a7d7a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f9ec44bdb2f34343b4a29ac371c30686, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/d69b7f91b7ed45a58d90655adf9b0412] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=35.2 K 2024-11-13T00:15:35,465 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a520a7f733774332ad14545df83a7d7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731456932323 2024-11-13T00:15:35,465 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c57024559f450fa31afd2fb38d202a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731456932323 2024-11-13T00:15:35,466 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ec44bdb2f34343b4a29ac371c30686, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731456932384 2024-11-13T00:15:35,466 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa98abe6d43b4d5e9e16a327f7e8572e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731456932384 2024-11-13T00:15:35,466 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0622314387c74ccdabd8ab8f8f15ad21, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731456934598 2024-11-13T00:15:35,466 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d69b7f91b7ed45a58d90655adf9b0412, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731456934598 2024-11-13T00:15:35,473 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:35,474 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ad366221e1fe4d30b7df74449be988ae is 50, key is test_row_0/A:col10/1731456934604/Put/seqid=0 2024-11-13T00:15:35,477 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:35,478 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/a52fda3de44d418bab29929fbfc2449a is 50, key is test_row_0/B:col10/1731456934604/Put/seqid=0 2024-11-13T00:15:35,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742331_1507 (size=12104) 2024-11-13T00:15:35,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742332_1508 (size=12104) 2024-11-13T00:15:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:35,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-13T00:15:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:35,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:35,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b2845232dff6448e9c2eb2cc95bd7d11 is 50, key is test_row_0/A:col10/1731456935851/Put/seqid=0 2024-11-13T00:15:35,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742333_1509 (size=14341) 2024-11-13T00:15:35,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456995867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456995868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,883 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ad366221e1fe4d30b7df74449be988ae as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ad366221e1fe4d30b7df74449be988ae 2024-11-13T00:15:35,886 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into ad366221e1fe4d30b7df74449be988ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:35,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:35,887 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456935464; duration=0sec 2024-11-13T00:15:35,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:35,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:35,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:35,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:35,887 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:35,888 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:35,888 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f5ad3a3f34414cfabbe8005ebce16137, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/325d13bb83754e5caa7a94642fcc06ab, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/728cae57a56b4b749e6b904554034063] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=35.2 K 2024-11-13T00:15:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456995871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,888 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ad3a3f34414cfabbe8005ebce16137, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731456932323 2024-11-13T00:15:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,888 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325d13bb83754e5caa7a94642fcc06ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731456932384 2024-11-13T00:15:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456995876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456995876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,889 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 728cae57a56b4b749e6b904554034063, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731456934598 2024-11-13T00:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,891 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/a52fda3de44d418bab29929fbfc2449a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a52fda3de44d418bab29929fbfc2449a 2024-11-13T00:15:35,895 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into a52fda3de44d418bab29929fbfc2449a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:35,895 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:35,895 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456935464; duration=0sec 2024-11-13T00:15:35,895 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:35,895 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:35,897 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:35,897 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/e00f75a4e3214c68a870d3d56e233b2e is 50, key is test_row_0/C:col10/1731456934604/Put/seqid=0 2024-11-13T00:15:35,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742334_1510 (size=12104) 2024-11-13T00:15:35,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456995985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456995986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456995996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456996205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456996206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456996206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b2845232dff6448e9c2eb2cc95bd7d11 2024-11-13T00:15:36,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/bf0a95dd7db44382a7ebe155b984f782 is 50, key is test_row_0/B:col10/1731456935851/Put/seqid=0 2024-11-13T00:15:36,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742335_1511 (size=12001) 2024-11-13T00:15:36,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/bf0a95dd7db44382a7ebe155b984f782 2024-11-13T00:15:36,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d5a609f630ef49738a29fa80e050d53b is 50, key is test_row_0/C:col10/1731456935851/Put/seqid=0 2024-11-13T00:15:36,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742336_1512 (size=12001) 2024-11-13T00:15:36,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d5a609f630ef49738a29fa80e050d53b 2024-11-13T00:15:36,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b2845232dff6448e9c2eb2cc95bd7d11 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b2845232dff6448e9c2eb2cc95bd7d11 2024-11-13T00:15:36,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b2845232dff6448e9c2eb2cc95bd7d11, entries=200, sequenceid=93, filesize=14.0 K 2024-11-13T00:15:36,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/bf0a95dd7db44382a7ebe155b984f782 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/bf0a95dd7db44382a7ebe155b984f782 2024-11-13T00:15:36,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/bf0a95dd7db44382a7ebe155b984f782, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T00:15:36,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d5a609f630ef49738a29fa80e050d53b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d5a609f630ef49738a29fa80e050d53b 2024-11-13T00:15:36,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d5a609f630ef49738a29fa80e050d53b, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T00:15:36,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for a66cd1bb022d966edc952b066b868a43 in 453ms, sequenceid=93, compaction requested=false 2024-11-13T00:15:36,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:36,306 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/e00f75a4e3214c68a870d3d56e233b2e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e00f75a4e3214c68a870d3d56e233b2e 2024-11-13T00:15:36,311 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into e00f75a4e3214c68a870d3d56e233b2e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:36,311 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:36,311 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=13, startTime=1731456935464; duration=0sec 2024-11-13T00:15:36,311 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:36,311 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T00:15:36,476 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:36,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T00:15:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:36,478 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:36,479 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:36,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:36,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:36,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc is 50, key is test_row_0/A:col10/1731456935860/Put/seqid=0 2024-11-13T00:15:36,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742337_1513 (size=14341) 2024-11-13T00:15:36,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc 2024-11-13T00:15:36,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/ac19545098614c93bf66fd086503353b is 50, key is test_row_0/B:col10/1731456935860/Put/seqid=0 2024-11-13T00:15:36,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742338_1514 (size=12001) 2024-11-13T00:15:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456936574, max timestamp=1731456936585Num uniques 859; 2024-11-13T00:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456936574, max timestamp=1731456936585Num uniques 859; 2024-11-13T00:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456936574, max timestamp=1731456936585Num uniques 859; 2024-11-13T00:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:36,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T00:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:36,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456996641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456996648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456996650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456996756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456996765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T00:15:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:36,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456996776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:36,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456996896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456996896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,904 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:36,905 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T00:15:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:36,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:36,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/ac19545098614c93bf66fd086503353b 2024-11-13T00:15:36,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/e1587781d39a4af2b5e7d1fce2eacae7 is 50, key is test_row_0/C:col10/1731456935860/Put/seqid=0 2024-11-13T00:15:36,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742339_1515 (size=12001) 2024-11-13T00:15:36,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456996976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456996976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456996996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T00:15:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:37,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T00:15:37,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:37,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:37,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456997296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456997296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456997316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/e1587781d39a4af2b5e7d1fce2eacae7 2024-11-13T00:15:37,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc 2024-11-13T00:15:37,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc, entries=200, sequenceid=106, filesize=14.0 K 2024-11-13T00:15:37,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/ac19545098614c93bf66fd086503353b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/ac19545098614c93bf66fd086503353b 2024-11-13T00:15:37,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/ac19545098614c93bf66fd086503353b, entries=150, sequenceid=106, filesize=11.7 K 2024-11-13T00:15:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/e1587781d39a4af2b5e7d1fce2eacae7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e1587781d39a4af2b5e7d1fce2eacae7 2024-11-13T00:15:37,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e1587781d39a4af2b5e7d1fce2eacae7, entries=150, sequenceid=106, filesize=11.7 K 2024-11-13T00:15:37,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a66cd1bb022d966edc952b066b868a43 in 858ms, sequenceid=106, compaction requested=true 2024-11-13T00:15:37,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:37,389 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:37,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:37,390 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:37,390 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:37,390 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:37,390 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,390 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ad366221e1fe4d30b7df74449be988ae, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b2845232dff6448e9c2eb2cc95bd7d11, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=39.8 K 2024-11-13T00:15:37,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad366221e1fe4d30b7df74449be988ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731456934598 2024-11-13T00:15:37,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:37,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:37,391 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,391 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a52fda3de44d418bab29929fbfc2449a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/bf0a95dd7db44382a7ebe155b984f782, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/ac19545098614c93bf66fd086503353b] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=35.3 K 2024-11-13T00:15:37,391 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2845232dff6448e9c2eb2cc95bd7d11, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731456934666 2024-11-13T00:15:37,391 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting a52fda3de44d418bab29929fbfc2449a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731456934598 2024-11-13T00:15:37,392 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a550103e1b4dcf9c49fa7b4a2ef0bc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731456935860 2024-11-13T00:15:37,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bf0a95dd7db44382a7ebe155b984f782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731456934666 2024-11-13T00:15:37,392 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ac19545098614c93bf66fd086503353b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731456935860 2024-11-13T00:15:37,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T00:15:37,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,395 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T00:15:37,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:37,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b8bf801b91844f96b99b68358ccd0700 is 50, key is test_row_0/A:col10/1731456936639/Put/seqid=0 2024-11-13T00:15:37,401 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:37,401 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:37,401 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/b843e714cf694a92b26df4b2254be88f is 50, key is test_row_0/B:col10/1731456935860/Put/seqid=0 2024-11-13T00:15:37,402 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/51be1f92211c49569d429ab0b4bb5f2a is 50, key is test_row_0/A:col10/1731456935860/Put/seqid=0 2024-11-13T00:15:37,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742340_1516 (size=12151) 2024-11-13T00:15:37,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742341_1517 (size=12207) 2024-11-13T00:15:37,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742342_1518 (size=12207) 2024-11-13T00:15:37,419 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/b843e714cf694a92b26df4b2254be88f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b843e714cf694a92b26df4b2254be88f 2024-11-13T00:15:37,425 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/51be1f92211c49569d429ab0b4bb5f2a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/51be1f92211c49569d429ab0b4bb5f2a 2024-11-13T00:15:37,427 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into b843e714cf694a92b26df4b2254be88f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:37,427 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:37,427 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456937390; duration=0sec 2024-11-13T00:15:37,427 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:37,427 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:37,427 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:37,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:37,428 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:37,428 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:37,428 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e00f75a4e3214c68a870d3d56e233b2e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d5a609f630ef49738a29fa80e050d53b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e1587781d39a4af2b5e7d1fce2eacae7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=35.3 K 2024-11-13T00:15:37,429 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e00f75a4e3214c68a870d3d56e233b2e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731456934598 2024-11-13T00:15:37,429 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a609f630ef49738a29fa80e050d53b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731456934666 2024-11-13T00:15:37,429 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e1587781d39a4af2b5e7d1fce2eacae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731456935860 2024-11-13T00:15:37,429 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into 51be1f92211c49569d429ab0b4bb5f2a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:37,429 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:37,430 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456937389; duration=0sec 2024-11-13T00:15:37,430 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:37,430 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:37,435 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:37,435 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4a5d2aaf60be4c81809e07e1986a203f is 50, key is test_row_0/C:col10/1731456935860/Put/seqid=0 2024-11-13T00:15:37,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742343_1519 (size=12207) 2024-11-13T00:15:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:37,813 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b8bf801b91844f96b99b68358ccd0700 2024-11-13T00:15:37,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/097129155f9a4fcb9e4819a0a2764564 is 50, key is test_row_0/B:col10/1731456936639/Put/seqid=0 2024-11-13T00:15:37,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742344_1520 (size=12151) 2024-11-13T00:15:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456997830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456997831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456997842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,850 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4a5d2aaf60be4c81809e07e1986a203f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4a5d2aaf60be4c81809e07e1986a203f 2024-11-13T00:15:37,854 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into 4a5d2aaf60be4c81809e07e1986a203f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:37,854 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:37,854 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=13, startTime=1731456937390; duration=0sec 2024-11-13T00:15:37,854 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:37,854 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456997946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456997946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456997946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456998166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456998166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456998166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:38,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,225 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/097129155f9a4fcb9e4819a0a2764564 2024-11-13T00:15:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/fc45f480e7234501a8e5246eddf6eda3 is 50, key is test_row_0/C:col10/1731456936639/Put/seqid=0 2024-11-13T00:15:38,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742345_1521 (size=12151) 2024-11-13T00:15:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456998476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456998476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456998486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:38,639 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/fc45f480e7234501a8e5246eddf6eda3 2024-11-13T00:15:38,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b8bf801b91844f96b99b68358ccd0700 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b8bf801b91844f96b99b68358ccd0700 2024-11-13T00:15:38,647 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b8bf801b91844f96b99b68358ccd0700, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T00:15:38,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/097129155f9a4fcb9e4819a0a2764564 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/097129155f9a4fcb9e4819a0a2764564 2024-11-13T00:15:38,651 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/097129155f9a4fcb9e4819a0a2764564, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T00:15:38,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/fc45f480e7234501a8e5246eddf6eda3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/fc45f480e7234501a8e5246eddf6eda3 2024-11-13T00:15:38,654 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/fc45f480e7234501a8e5246eddf6eda3, entries=150, sequenceid=143, filesize=11.9 K 2024-11-13T00:15:38,655 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a66cd1bb022d966edc952b066b868a43 in 1260ms, sequenceid=143, compaction requested=false 2024-11-13T00:15:38,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:38,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:38,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T00:15:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T00:15:38,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T00:15:38,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-13T00:15:38,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1830 sec 2024-11-13T00:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:38,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:38,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ea70dab339ec4e29b0439b962e301075 is 50, key is test_row_0/A:col10/1731456937840/Put/seqid=0 2024-11-13T00:15:38,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742346_1522 (size=14541) 2024-11-13T00:15:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731456998995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456938980, max timestamp=1731456939006Num uniques 859; 2024-11-13T00:15:39,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:39,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456938980, max timestamp=1731456939006Num uniques 859; 2024-11-13T00:15:39,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456938980, max timestamp=1731456939006Num uniques 859; 2024-11-13T00:15:39,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:39,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456999043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456999044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456999050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456999051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456999156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456999166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456999166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456999176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ea70dab339ec4e29b0439b962e301075 2024-11-13T00:15:39,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/7b359ba29cb44d3e9713598c91cf0711 is 50, key is test_row_0/B:col10/1731456937840/Put/seqid=0 2024-11-13T00:15:39,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742347_1523 (size=12151) 2024-11-13T00:15:39,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/7b359ba29cb44d3e9713598c91cf0711 2024-11-13T00:15:39,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/84440b50f617467ab9438d7bcc79f252 is 50, key is test_row_0/C:col10/1731456937840/Put/seqid=0 2024-11-13T00:15:39,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742348_1524 (size=12151) 2024-11-13T00:15:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456999376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456999386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456999386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456999396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731456999696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731456999706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731456999706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731456999716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:39,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/84440b50f617467ab9438d7bcc79f252 2024-11-13T00:15:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ea70dab339ec4e29b0439b962e301075 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ea70dab339ec4e29b0439b962e301075 2024-11-13T00:15:39,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ea70dab339ec4e29b0439b962e301075, entries=200, sequenceid=157, filesize=14.2 K 2024-11-13T00:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/7b359ba29cb44d3e9713598c91cf0711 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7b359ba29cb44d3e9713598c91cf0711 2024-11-13T00:15:39,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7b359ba29cb44d3e9713598c91cf0711, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T00:15:39,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/84440b50f617467ab9438d7bcc79f252 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/84440b50f617467ab9438d7bcc79f252 2024-11-13T00:15:39,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/84440b50f617467ab9438d7bcc79f252, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T00:15:39,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a66cd1bb022d966edc952b066b868a43 in 852ms, sequenceid=157, compaction requested=true 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:39,782 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:39,782 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:39,783 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:39,783 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:39,783 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:39,783 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b843e714cf694a92b26df4b2254be88f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/097129155f9a4fcb9e4819a0a2764564, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7b359ba29cb44d3e9713598c91cf0711] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=35.7 K 2024-11-13T00:15:39,783 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:39,783 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:39,783 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:39,783 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/51be1f92211c49569d429ab0b4bb5f2a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b8bf801b91844f96b99b68358ccd0700, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ea70dab339ec4e29b0439b962e301075] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=38.0 K 2024-11-13T00:15:39,784 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b843e714cf694a92b26df4b2254be88f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731456935860 2024-11-13T00:15:39,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51be1f92211c49569d429ab0b4bb5f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731456935860 2024-11-13T00:15:39,784 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 097129155f9a4fcb9e4819a0a2764564, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731456936639 2024-11-13T00:15:39,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8bf801b91844f96b99b68358ccd0700, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731456936639 2024-11-13T00:15:39,784 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea70dab339ec4e29b0439b962e301075, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731456937815 2024-11-13T00:15:39,784 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b359ba29cb44d3e9713598c91cf0711, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731456937830 2024-11-13T00:15:39,790 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:39,791 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/8effb56f501b4f02a75620e69cc91ff4 is 50, key is test_row_0/B:col10/1731456937840/Put/seqid=0 2024-11-13T00:15:39,793 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:39,794 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8e530db524b84d57a6f2f38754729f3c is 50, key is test_row_0/A:col10/1731456937840/Put/seqid=0 2024-11-13T00:15:39,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742349_1525 (size=12459) 2024-11-13T00:15:39,808 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/8effb56f501b4f02a75620e69cc91ff4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/8effb56f501b4f02a75620e69cc91ff4 2024-11-13T00:15:39,816 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into 8effb56f501b4f02a75620e69cc91ff4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:39,816 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:39,816 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456939782; duration=0sec 2024-11-13T00:15:39,816 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:39,816 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:39,816 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:39,818 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:39,818 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:39,818 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:39,818 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4a5d2aaf60be4c81809e07e1986a203f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/fc45f480e7234501a8e5246eddf6eda3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/84440b50f617467ab9438d7bcc79f252] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=35.7 K 2024-11-13T00:15:39,819 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5d2aaf60be4c81809e07e1986a203f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731456935860 2024-11-13T00:15:39,819 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting fc45f480e7234501a8e5246eddf6eda3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731456936639 2024-11-13T00:15:39,820 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 84440b50f617467ab9438d7bcc79f252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731456937830 2024-11-13T00:15:39,832 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:39,833 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/145007e812c54d0e814bb53dccdbe5c5 is 50, key is test_row_0/C:col10/1731456937840/Put/seqid=0 2024-11-13T00:15:39,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742350_1526 (size=12459) 2024-11-13T00:15:39,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742351_1527 (size=12459) 2024-11-13T00:15:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:40,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T00:15:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:40,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:40,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8b410fd98de946ae8809c5ab757bca61 is 50, key is test_row_0/A:col10/1731456939049/Put/seqid=0 2024-11-13T00:15:40,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742352_1528 (size=14541) 2024-11-13T00:15:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457000078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457000195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731457000206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731457000226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731457000226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,242 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8e530db524b84d57a6f2f38754729f3c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8e530db524b84d57a6f2f38754729f3c 2024-11-13T00:15:40,246 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into 8e530db524b84d57a6f2f38754729f3c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:40,246 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:40,246 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456939782; duration=0sec 2024-11-13T00:15:40,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,246 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:40,246 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731457000235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,249 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/145007e812c54d0e814bb53dccdbe5c5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/145007e812c54d0e814bb53dccdbe5c5 2024-11-13T00:15:40,252 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into 145007e812c54d0e814bb53dccdbe5c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:40,252 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:40,252 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=13, startTime=1731456939782; duration=0sec 2024-11-13T00:15:40,252 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:40,252 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457000406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8b410fd98de946ae8809c5ab757bca61 2024-11-13T00:15:40,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/29089fbd1ec54efa8a7f2a3b1be5b0ff is 50, key is test_row_0/B:col10/1731456939049/Put/seqid=0 2024-11-13T00:15:40,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742353_1529 (size=12151) 2024-11-13T00:15:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T00:15:40,646 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:40,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T00:15:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T00:15:40,648 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:40,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:40,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:40,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457000726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T00:15:40,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T00:15:40,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:40,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:40,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:40,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:40,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:40,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/29089fbd1ec54efa8a7f2a3b1be5b0ff 2024-11-13T00:15:40,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/420bfdba7db94a629a905aa2a56ee816 is 50, key is test_row_0/C:col10/1731456939049/Put/seqid=0 2024-11-13T00:15:40,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742354_1530 (size=12151) 2024-11-13T00:15:40,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T00:15:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:40,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T00:15:41,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T00:15:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:41,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:41,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731457001226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:41,232 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:41,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457001236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:41,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731457001245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731457001246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:41,252 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:41,252 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:41,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T00:15:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:41,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:41,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/420bfdba7db94a629a905aa2a56ee816 2024-11-13T00:15:41,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8b410fd98de946ae8809c5ab757bca61 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8b410fd98de946ae8809c5ab757bca61 2024-11-13T00:15:41,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8b410fd98de946ae8809c5ab757bca61, entries=200, sequenceid=194, filesize=14.2 K 2024-11-13T00:15:41,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/29089fbd1ec54efa8a7f2a3b1be5b0ff as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/29089fbd1ec54efa8a7f2a3b1be5b0ff 2024-11-13T00:15:41,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/29089fbd1ec54efa8a7f2a3b1be5b0ff, entries=150, sequenceid=194, filesize=11.9 K 2024-11-13T00:15:41,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/420bfdba7db94a629a905aa2a56ee816 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/420bfdba7db94a629a905aa2a56ee816 2024-11-13T00:15:41,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731457001266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:41,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/420bfdba7db94a629a905aa2a56ee816, entries=150, sequenceid=194, filesize=11.9 K 2024-11-13T00:15:41,276 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:41,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:41,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a66cd1bb022d966edc952b066b868a43 in 1255ms, sequenceid=194, compaction requested=false 2024-11-13T00:15:41,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T00:15:41,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:41,413 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:41,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c5e3628547db4ca6b81b0bd8da1e6fd9 is 50, key is test_row_0/A:col10/1731456940063/Put/seqid=0 2024-11-13T00:15:41,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742355_1531 (size=12151) 2024-11-13T00:15:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T00:15:41,823 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c5e3628547db4ca6b81b0bd8da1e6fd9 2024-11-13T00:15:41,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/9d9efe7f923d448abf022e5ecf034630 is 50, key is test_row_0/B:col10/1731456940063/Put/seqid=0 2024-11-13T00:15:41,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742356_1532 (size=12151) 2024-11-13T00:15:42,233 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/9d9efe7f923d448abf022e5ecf034630 2024-11-13T00:15:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/cbf66ceee5f141609bc5681b644e4b84 is 50, key is test_row_0/C:col10/1731456940063/Put/seqid=0 2024-11-13T00:15:42,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742357_1533 (size=12151) 2024-11-13T00:15:42,243 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/cbf66ceee5f141609bc5681b644e4b84 2024-11-13T00:15:42,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c5e3628547db4ca6b81b0bd8da1e6fd9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c5e3628547db4ca6b81b0bd8da1e6fd9 2024-11-13T00:15:42,250 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c5e3628547db4ca6b81b0bd8da1e6fd9, entries=150, sequenceid=206, filesize=11.9 K 2024-11-13T00:15:42,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/9d9efe7f923d448abf022e5ecf034630 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/9d9efe7f923d448abf022e5ecf034630 2024-11-13T00:15:42,253 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/9d9efe7f923d448abf022e5ecf034630, entries=150, sequenceid=206, filesize=11.9 K 2024-11-13T00:15:42,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/cbf66ceee5f141609bc5681b644e4b84 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/cbf66ceee5f141609bc5681b644e4b84 2024-11-13T00:15:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:42,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:42,261 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/cbf66ceee5f141609bc5681b644e4b84, entries=150, sequenceid=206, filesize=11.9 K 2024-11-13T00:15:42,262 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for a66cd1bb022d966edc952b066b868a43 in 849ms, sequenceid=206, compaction requested=true 2024-11-13T00:15:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T00:15:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T00:15:42,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T00:15:42,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-13T00:15:42,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6200 sec 2024-11-13T00:15:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:42,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:42,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:42,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:42,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:42,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:42,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:42,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:42,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ff8226046a59410da5963d917a1e1bd7 is 50, key is test_row_1/A:col10/1731456942274/Put/seqid=0 2024-11-13T00:15:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742358_1534 (size=12147) 2024-11-13T00:15:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456942388, max timestamp=1731456942421Num uniques 859; 2024-11-13T00:15:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456942388, max timestamp=1731456942421Num uniques 859; 2024-11-13T00:15:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456942388, max timestamp=1731456942421Num uniques 859; 2024-11-13T00:15:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:42,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457002534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:42,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457002656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:42,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ff8226046a59410da5963d917a1e1bd7 2024-11-13T00:15:42,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/c9f6caf7fbb2454ebad63cad6cbd5159 is 50, key is test_row_1/B:col10/1731456942274/Put/seqid=0 2024-11-13T00:15:42,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742359_1535 (size=9757) 2024-11-13T00:15:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T00:15:42,796 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:42,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T00:15:42,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T00:15:42,800 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:42,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457002876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T00:15:42,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T00:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:42,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T00:15:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/c9f6caf7fbb2454ebad63cad6cbd5159 2024-11-13T00:15:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T00:15:43,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4eba524c432c486da6b6e7ee8c937b58 is 50, key is test_row_1/C:col10/1731456942274/Put/seqid=0 2024-11-13T00:15:43,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742360_1536 (size=9757) 2024-11-13T00:15:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457003195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731457003246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:43,254 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:43,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T00:15:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731457003266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:43,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731457003266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:43,278 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:43,278 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:43,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731457003296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:43,309 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:43,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:43,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:43,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T00:15:43,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4eba524c432c486da6b6e7ee8c937b58 2024-11-13T00:15:43,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/ff8226046a59410da5963d917a1e1bd7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ff8226046a59410da5963d917a1e1bd7 2024-11-13T00:15:43,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ff8226046a59410da5963d917a1e1bd7, entries=150, sequenceid=216, filesize=11.9 K 2024-11-13T00:15:43,527 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-13T00:15:43,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/c9f6caf7fbb2454ebad63cad6cbd5159 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/c9f6caf7fbb2454ebad63cad6cbd5159 2024-11-13T00:15:43,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/c9f6caf7fbb2454ebad63cad6cbd5159, entries=100, sequenceid=216, filesize=9.5 K 2024-11-13T00:15:43,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4eba524c432c486da6b6e7ee8c937b58 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4eba524c432c486da6b6e7ee8c937b58 2024-11-13T00:15:43,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4eba524c432c486da6b6e7ee8c937b58, entries=100, sequenceid=216, filesize=9.5 K 2024-11-13T00:15:43,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a66cd1bb022d966edc952b066b868a43 in 1252ms, sequenceid=216, compaction requested=true 2024-11-13T00:15:43,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:43,536 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:43,536 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:43,537 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:43,537 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:43,538 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:43,538 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:43,538 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,538 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,538 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8e530db524b84d57a6f2f38754729f3c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8b410fd98de946ae8809c5ab757bca61, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c5e3628547db4ca6b81b0bd8da1e6fd9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ff8226046a59410da5963d917a1e1bd7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=50.1 K 2024-11-13T00:15:43,538 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/8effb56f501b4f02a75620e69cc91ff4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/29089fbd1ec54efa8a7f2a3b1be5b0ff, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/9d9efe7f923d448abf022e5ecf034630, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/c9f6caf7fbb2454ebad63cad6cbd5159] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=45.4 K 2024-11-13T00:15:43,538 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e530db524b84d57a6f2f38754729f3c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731456937830 2024-11-13T00:15:43,538 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 8effb56f501b4f02a75620e69cc91ff4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731456937830 2024-11-13T00:15:43,538 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b410fd98de946ae8809c5ab757bca61, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731456939041 2024-11-13T00:15:43,538 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 29089fbd1ec54efa8a7f2a3b1be5b0ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731456939048 2024-11-13T00:15:43,539 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e3628547db4ca6b81b0bd8da1e6fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731456940053 2024-11-13T00:15:43,539 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9efe7f923d448abf022e5ecf034630, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731456940053 2024-11-13T00:15:43,539 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff8226046a59410da5963d917a1e1bd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456942265 2024-11-13T00:15:43,539 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f6caf7fbb2454ebad63cad6cbd5159, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456942274 2024-11-13T00:15:43,548 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:43,549 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/60a7cd4b7e224cbda5410f43612b6f7a is 50, key is test_row_0/B:col10/1731456940063/Put/seqid=0 2024-11-13T00:15:43,552 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:43,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742361_1537 (size=12595) 2024-11-13T00:15:43,552 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/bb8653662aea4cdbad448204d712594c is 50, key is test_row_0/A:col10/1731456940063/Put/seqid=0 2024-11-13T00:15:43,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,566 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:43,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:43,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742362_1538 (size=12595) 2024-11-13T00:15:43,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/1eda4151c1e1416685ca4027de16ff40 is 50, key is test_row_0/A:col10/1731456942527/Put/seqid=0 2024-11-13T00:15:43,572 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/bb8653662aea4cdbad448204d712594c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bb8653662aea4cdbad448204d712594c 2024-11-13T00:15:43,577 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into bb8653662aea4cdbad448204d712594c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:43,577 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:43,577 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=12, startTime=1731456943535; duration=0sec 2024-11-13T00:15:43,577 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:43,577 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:43,578 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:43,578 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:43,580 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:43,580 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,580 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/145007e812c54d0e814bb53dccdbe5c5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/420bfdba7db94a629a905aa2a56ee816, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/cbf66ceee5f141609bc5681b644e4b84, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4eba524c432c486da6b6e7ee8c937b58] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=45.4 K 2024-11-13T00:15:43,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742363_1539 (size=12151) 2024-11-13T00:15:43,583 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145007e812c54d0e814bb53dccdbe5c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731456937830 2024-11-13T00:15:43,583 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/1eda4151c1e1416685ca4027de16ff40 2024-11-13T00:15:43,583 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420bfdba7db94a629a905aa2a56ee816, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731456939048 2024-11-13T00:15:43,583 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf66ceee5f141609bc5681b644e4b84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731456940053 2024-11-13T00:15:43,584 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eba524c432c486da6b6e7ee8c937b58, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456942274 2024-11-13T00:15:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/216939ce6cc4462689077fd762e00acc is 50, key is test_row_0/B:col10/1731456942527/Put/seqid=0 2024-11-13T00:15:43,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742364_1540 (size=12151) 2024-11-13T00:15:43,593 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/216939ce6cc4462689077fd762e00acc 2024-11-13T00:15:43,593 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:43,594 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/bd7ca74381fe4e15b0290b0899c51cd7 is 50, key is test_row_0/C:col10/1731456940063/Put/seqid=0 2024-11-13T00:15:43,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742365_1541 (size=12595) 2024-11-13T00:15:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/3896ad3862b84407a19c471877d84be7 is 50, key is test_row_0/C:col10/1731456942527/Put/seqid=0 2024-11-13T00:15:43,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742366_1542 (size=12151) 2024-11-13T00:15:43,604 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/3896ad3862b84407a19c471877d84be7 2024-11-13T00:15:43,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/1eda4151c1e1416685ca4027de16ff40 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/1eda4151c1e1416685ca4027de16ff40 2024-11-13T00:15:43,611 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/1eda4151c1e1416685ca4027de16ff40, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T00:15:43,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/216939ce6cc4462689077fd762e00acc as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/216939ce6cc4462689077fd762e00acc 2024-11-13T00:15:43,615 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/216939ce6cc4462689077fd762e00acc, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T00:15:43,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/3896ad3862b84407a19c471877d84be7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3896ad3862b84407a19c471877d84be7 2024-11-13T00:15:43,619 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3896ad3862b84407a19c471877d84be7, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T00:15:43,621 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for a66cd1bb022d966edc952b066b868a43 in 54ms, sequenceid=253, compaction requested=false 2024-11-13T00:15:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T00:15:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T00:15:43,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T00:15:43,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 822 msec 2024-11-13T00:15:43,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 827 msec 2024-11-13T00:15:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:43,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:43,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:43,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/a5e63583e91043c18b6f59d6a4751b81 is 50, key is test_row_0/A:col10/1731456943731/Put/seqid=0 2024-11-13T00:15:43,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742367_1543 (size=12297) 2024-11-13T00:15:43,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456943833, max timestamp=1731456943865Num uniques 859; 2024-11-13T00:15:43,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:43,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456943833, max timestamp=1731456943865Num uniques 859; 2024-11-13T00:15:43,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:43,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456943833, max timestamp=1731456943865Num uniques 859; 2024-11-13T00:15:43,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T00:15:43,936 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:43,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T00:15:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T00:15:43,938 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:43,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:43,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:43,957 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/60a7cd4b7e224cbda5410f43612b6f7a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/60a7cd4b7e224cbda5410f43612b6f7a 2024-11-13T00:15:43,961 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into 60a7cd4b7e224cbda5410f43612b6f7a(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:43,961 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:43,961 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=12, startTime=1731456943536; duration=0sec 2024-11-13T00:15:43,961 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:43,961 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:44,003 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/bd7ca74381fe4e15b0290b0899c51cd7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bd7ca74381fe4e15b0290b0899c51cd7 2024-11-13T00:15:44,006 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into bd7ca74381fe4e15b0290b0899c51cd7(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:44,006 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:44,006 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=12, startTime=1731456943536; duration=0sec 2024-11-13T00:15:44,006 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:44,006 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:44,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457004027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:44,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T00:15:44,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/a5e63583e91043c18b6f59d6a4751b81 2024-11-13T00:15:44,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f3110799eaea4488b69218e489213d40 is 50, key is test_row_0/B:col10/1731456943731/Put/seqid=0 2024-11-13T00:15:44,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457004146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:44,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742368_1544 (size=9857) 2024-11-13T00:15:44,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T00:15:44,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457004366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:44,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:44,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f3110799eaea4488b69218e489213d40 2024-11-13T00:15:44,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/a3ac57c8a6b14e019a1279a74cefe4e6 is 50, key is test_row_0/C:col10/1731456943731/Put/seqid=0 2024-11-13T00:15:44,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742369_1545 (size=9857) 2024-11-13T00:15:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T00:15:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457004686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:44,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:44,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:44,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:44,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/a3ac57c8a6b14e019a1279a74cefe4e6 2024-11-13T00:15:44,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/a5e63583e91043c18b6f59d6a4751b81 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/a5e63583e91043c18b6f59d6a4751b81 2024-11-13T00:15:44,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/a5e63583e91043c18b6f59d6a4751b81, entries=150, sequenceid=264, filesize=12.0 K 2024-11-13T00:15:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f3110799eaea4488b69218e489213d40 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f3110799eaea4488b69218e489213d40 2024-11-13T00:15:44,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f3110799eaea4488b69218e489213d40, entries=100, sequenceid=264, filesize=9.6 K 2024-11-13T00:15:44,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/a3ac57c8a6b14e019a1279a74cefe4e6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a3ac57c8a6b14e019a1279a74cefe4e6 2024-11-13T00:15:44,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a3ac57c8a6b14e019a1279a74cefe4e6, entries=100, sequenceid=264, filesize=9.6 K 2024-11-13T00:15:44,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a66cd1bb022d966edc952b066b868a43 in 1248ms, sequenceid=264, compaction requested=true 2024-11-13T00:15:44,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:44,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:44,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:44,980 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:44,981 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:44,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:44,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:44,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:44,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:44,981 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:44,981 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:44,981 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,981 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bb8653662aea4cdbad448204d712594c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/1eda4151c1e1416685ca4027de16ff40, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/a5e63583e91043c18b6f59d6a4751b81] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.2 K 2024-11-13T00:15:44,982 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:44,982 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:44,982 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:44,982 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/60a7cd4b7e224cbda5410f43612b6f7a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/216939ce6cc4462689077fd762e00acc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f3110799eaea4488b69218e489213d40] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=33.8 K 2024-11-13T00:15:44,982 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb8653662aea4cdbad448204d712594c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456940063 2024-11-13T00:15:44,983 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a7cd4b7e224cbda5410f43612b6f7a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456940063 2024-11-13T00:15:44,983 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eda4151c1e1416685ca4027de16ff40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731456942517 2024-11-13T00:15:44,983 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 216939ce6cc4462689077fd762e00acc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731456942517 2024-11-13T00:15:44,983 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e63583e91043c18b6f59d6a4751b81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731456943722 2024-11-13T00:15:44,983 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f3110799eaea4488b69218e489213d40, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731456943725 2024-11-13T00:15:44,992 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:44,992 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/5b0c745d3df44c189a75ff06efa2a8ab is 50, key is test_row_0/A:col10/1731456943731/Put/seqid=0 2024-11-13T00:15:44,993 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:44,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/321e9a1d37364be98e0fbe2b62958862 is 50, key is test_row_0/B:col10/1731456943731/Put/seqid=0 2024-11-13T00:15:45,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:45,009 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:45,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742370_1546 (size=12797) 2024-11-13T00:15:45,019 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/5b0c745d3df44c189a75ff06efa2a8ab as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/5b0c745d3df44c189a75ff06efa2a8ab 2024-11-13T00:15:45,024 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into 5b0c745d3df44c189a75ff06efa2a8ab(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:45,024 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:45,024 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456944980; duration=0sec 2024-11-13T00:15:45,024 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:45,024 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:45,024 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/0e0572939d3242b3b94a803272102293 is 50, key is test_row_0/A:col10/1731456943983/Put/seqid=0 2024-11-13T00:15:45,026 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:45,027 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:45,027 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:45,027 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bd7ca74381fe4e15b0290b0899c51cd7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3896ad3862b84407a19c471877d84be7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a3ac57c8a6b14e019a1279a74cefe4e6] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=33.8 K 2024-11-13T00:15:45,027 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd7ca74381fe4e15b0290b0899c51cd7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731456940063 2024-11-13T00:15:45,027 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3896ad3862b84407a19c471877d84be7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731456942517 2024-11-13T00:15:45,028 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ac57c8a6b14e019a1279a74cefe4e6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731456943725 2024-11-13T00:15:45,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742371_1547 (size=12797) 2024-11-13T00:15:45,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742372_1548 (size=12301) 2024-11-13T00:15:45,034 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/0e0572939d3242b3b94a803272102293 2024-11-13T00:15:45,037 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/321e9a1d37364be98e0fbe2b62958862 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/321e9a1d37364be98e0fbe2b62958862 2024-11-13T00:15:45,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/03144b76aa4d4935936553d76922cbbc is 50, key is test_row_0/B:col10/1731456943983/Put/seqid=0 2024-11-13T00:15:45,044 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into 321e9a1d37364be98e0fbe2b62958862(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:45,044 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:45,044 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456944980; duration=0sec 2024-11-13T00:15:45,044 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:45,044 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:45,054 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:45,055 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d3770d7faaa742a192da7f117f769f10 is 50, key is test_row_0/C:col10/1731456943731/Put/seqid=0 2024-11-13T00:15:45,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742373_1549 (size=12301) 2024-11-13T00:15:45,057 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/03144b76aa4d4935936553d76922cbbc 2024-11-13T00:15:45,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742374_1550 (size=12797) 2024-11-13T00:15:45,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/f8e484ccd824416e8c335b9c9dc2d465 is 50, key is test_row_0/C:col10/1731456943983/Put/seqid=0 2024-11-13T00:15:45,068 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d3770d7faaa742a192da7f117f769f10 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d3770d7faaa742a192da7f117f769f10 2024-11-13T00:15:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742375_1551 (size=12301) 2024-11-13T00:15:45,075 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into d3770d7faaa742a192da7f117f769f10(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:45,075 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:45,075 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=13, startTime=1731456944981; duration=0sec 2024-11-13T00:15:45,075 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:45,075 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T00:15:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:45,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457005260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457005376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:45,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:45,471 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/f8e484ccd824416e8c335b9c9dc2d465 2024-11-13T00:15:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/0e0572939d3242b3b94a803272102293 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0e0572939d3242b3b94a803272102293 2024-11-13T00:15:45,478 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0e0572939d3242b3b94a803272102293, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T00:15:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/03144b76aa4d4935936553d76922cbbc as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/03144b76aa4d4935936553d76922cbbc 2024-11-13T00:15:45,483 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/03144b76aa4d4935936553d76922cbbc, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T00:15:45,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/f8e484ccd824416e8c335b9c9dc2d465 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f8e484ccd824416e8c335b9c9dc2d465 2024-11-13T00:15:45,488 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f8e484ccd824416e8c335b9c9dc2d465, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T00:15:45,490 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a66cd1bb022d966edc952b066b868a43 in 480ms, sequenceid=303, compaction requested=false 2024-11-13T00:15:45,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:45,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:45,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T00:15:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T00:15:45,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T00:15:45,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5520 sec 2024-11-13T00:15:45,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5560 sec 2024-11-13T00:15:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:45,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:15:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:45,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:45,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:45,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:45,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:45,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:45,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/fac91fef06804273ada092403671ca27 is 50, key is test_row_0/A:col10/1731456945259/Put/seqid=0 2024-11-13T00:15:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742376_1552 (size=14741) 2024-11-13T00:15:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456945711, max timestamp=1731456945758Num uniques 859; 2024-11-13T00:15:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456945711, max timestamp=1731456945758Num uniques 859; 2024-11-13T00:15:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456945711, max timestamp=1731456945758Num uniques 859; 2024-11-13T00:15:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:45,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457005908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:46,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/fac91fef06804273ada092403671ca27 2024-11-13T00:15:46,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f7222acb708748f7928b24bf9b98a6b6 is 50, key is test_row_0/B:col10/1731456945259/Put/seqid=0 2024-11-13T00:15:46,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742377_1553 (size=12301) 2024-11-13T00:15:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457006026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:46,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T00:15:46,086 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:46,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-13T00:15:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T00:15:46,088 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:46,089 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:46,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T00:15:46,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T00:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457006246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:46,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T00:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T00:15:46,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f7222acb708748f7928b24bf9b98a6b6 2024-11-13T00:15:46,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/5dfbd1aa6e1b49f0b6f503647a734df1 is 50, key is test_row_0/C:col10/1731456945259/Put/seqid=0 2024-11-13T00:15:46,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742378_1554 (size=12301) 2024-11-13T00:15:46,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T00:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457006565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:46,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T00:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T00:15:46,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/5dfbd1aa6e1b49f0b6f503647a734df1 2024-11-13T00:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/fac91fef06804273ada092403671ca27 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fac91fef06804273ada092403671ca27 2024-11-13T00:15:46,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fac91fef06804273ada092403671ca27, entries=200, sequenceid=317, filesize=14.4 K 2024-11-13T00:15:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/f7222acb708748f7928b24bf9b98a6b6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f7222acb708748f7928b24bf9b98a6b6 2024-11-13T00:15:46,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f7222acb708748f7928b24bf9b98a6b6, entries=150, sequenceid=317, filesize=12.0 K 2024-11-13T00:15:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/5dfbd1aa6e1b49f0b6f503647a734df1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5dfbd1aa6e1b49f0b6f503647a734df1 2024-11-13T00:15:46,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5dfbd1aa6e1b49f0b6f503647a734df1, entries=150, sequenceid=317, filesize=12.0 K 2024-11-13T00:15:46,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a66cd1bb022d966edc952b066b868a43 in 1248ms, sequenceid=317, compaction requested=true 2024-11-13T00:15:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:46,838 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:46,838 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:46,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:46,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:46,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:46,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:46,839 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:46,839 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:46,839 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:46,839 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:46,839 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,839 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,839 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/5b0c745d3df44c189a75ff06efa2a8ab, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0e0572939d3242b3b94a803272102293, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fac91fef06804273ada092403671ca27] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=38.9 K 2024-11-13T00:15:46,839 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/321e9a1d37364be98e0fbe2b62958862, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/03144b76aa4d4935936553d76922cbbc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f7222acb708748f7928b24bf9b98a6b6] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.5 K 2024-11-13T00:15:46,840 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b0c745d3df44c189a75ff06efa2a8ab, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731456942517 2024-11-13T00:15:46,840 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 321e9a1d37364be98e0fbe2b62958862, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731456942517 2024-11-13T00:15:46,840 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 03144b76aa4d4935936553d76922cbbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731456943983 2024-11-13T00:15:46,840 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0572939d3242b3b94a803272102293, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731456943983 2024-11-13T00:15:46,840 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac91fef06804273ada092403671ca27, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731456945236 2024-11-13T00:15:46,840 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f7222acb708748f7928b24bf9b98a6b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731456945236 2024-11-13T00:15:46,846 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:46,846 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:46,847 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/e07b899f0f6645f9957614a26153cfba is 50, key is test_row_0/B:col10/1731456945259/Put/seqid=0 2024-11-13T00:15:46,847 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c8740325c96a487d98e73b265714319a is 50, key is test_row_0/A:col10/1731456945259/Put/seqid=0 2024-11-13T00:15:46,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742380_1556 (size=12949) 2024-11-13T00:15:46,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742379_1555 (size=12949) 2024-11-13T00:15:46,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T00:15:46,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,853 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T00:15:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:46,856 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/e07b899f0f6645f9957614a26153cfba as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/e07b899f0f6645f9957614a26153cfba 2024-11-13T00:15:46,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/eb38c8fc4ce641e38de7152ca62bf23d is 50, key is test_row_0/A:col10/1731456945881/Put/seqid=0 2024-11-13T00:15:46,860 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into e07b899f0f6645f9957614a26153cfba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:46,860 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:46,860 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456946838; duration=0sec 2024-11-13T00:15:46,860 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:46,860 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:46,860 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:46,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742381_1557 (size=12301) 2024-11-13T00:15:46,861 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:46,861 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:46,861 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:46,861 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d3770d7faaa742a192da7f117f769f10, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f8e484ccd824416e8c335b9c9dc2d465, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5dfbd1aa6e1b49f0b6f503647a734df1] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.5 K 2024-11-13T00:15:46,861 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting d3770d7faaa742a192da7f117f769f10, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731456942517 2024-11-13T00:15:46,862 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e484ccd824416e8c335b9c9dc2d465, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731456943983 2024-11-13T00:15:46,864 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfbd1aa6e1b49f0b6f503647a734df1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731456945236 2024-11-13T00:15:46,871 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:46,872 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/dc8720697c81473aaf81ba6f8e4c21b0 is 50, key is test_row_0/C:col10/1731456945259/Put/seqid=0 2024-11-13T00:15:46,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742382_1558 (size=12949) 2024-11-13T00:15:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:47,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457007163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T00:15:47,256 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c8740325c96a487d98e73b265714319a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8740325c96a487d98e73b265714319a 2024-11-13T00:15:47,259 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into c8740325c96a487d98e73b265714319a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:47,259 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:47,259 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456946838; duration=0sec 2024-11-13T00:15:47,259 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:47,259 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:47,261 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/eb38c8fc4ce641e38de7152ca62bf23d 2024-11-13T00:15:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/6dffc6f6c6aa40a8911f4e93cd9ec904 is 50, key is test_row_0/B:col10/1731456945881/Put/seqid=0 2024-11-13T00:15:47,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742383_1559 (size=12301) 2024-11-13T00:15:47,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457007276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,284 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/dc8720697c81473aaf81ba6f8e4c21b0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/dc8720697c81473aaf81ba6f8e4c21b0 2024-11-13T00:15:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,288 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into dc8720697c81473aaf81ba6f8e4c21b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:47,288 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:47,288 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=13, startTime=1731456946839; duration=0sec 2024-11-13T00:15:47,288 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:47,288 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59292 deadline: 1731457007296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,304 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59260 deadline: 1731457007305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59278 deadline: 1731457007306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,315 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,316 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59200 deadline: 1731457007346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,356 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in a66cd1bb022d966edc952b066b868a43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8305 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:15:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457007486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:47,671 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/6dffc6f6c6aa40a8911f4e93cd9ec904 2024-11-13T00:15:47,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/0b4be2f6fcb7456ead2bba9d07e1a4e9 is 50, key is test_row_0/C:col10/1731456945881/Put/seqid=0 2024-11-13T00:15:47,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742384_1560 (size=12301) 2024-11-13T00:15:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457007805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:48,082 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/0b4be2f6fcb7456ead2bba9d07e1a4e9 2024-11-13T00:15:48,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/eb38c8fc4ce641e38de7152ca62bf23d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/eb38c8fc4ce641e38de7152ca62bf23d 2024-11-13T00:15:48,089 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/eb38c8fc4ce641e38de7152ca62bf23d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-13T00:15:48,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/6dffc6f6c6aa40a8911f4e93cd9ec904 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6dffc6f6c6aa40a8911f4e93cd9ec904 2024-11-13T00:15:48,092 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6dffc6f6c6aa40a8911f4e93cd9ec904, entries=150, sequenceid=352, filesize=12.0 K 2024-11-13T00:15:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/0b4be2f6fcb7456ead2bba9d07e1a4e9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/0b4be2f6fcb7456ead2bba9d07e1a4e9 2024-11-13T00:15:48,096 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/0b4be2f6fcb7456ead2bba9d07e1a4e9, entries=150, sequenceid=352, filesize=12.0 K 2024-11-13T00:15:48,097 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a66cd1bb022d966edc952b066b868a43 in 1244ms, sequenceid=352, compaction requested=false 2024-11-13T00:15:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-13T00:15:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-13T00:15:48,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T00:15:48,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0090 sec 2024-11-13T00:15:48,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0150 sec 2024-11-13T00:15:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T00:15:48,236 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:48,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-13T00:15:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T00:15:48,238 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:48,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:48,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:48,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:48,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/79cddd5b7a4447edb4da022ffe196cfe is 50, key is test_row_0/A:col10/1731456947162/Put/seqid=0 2024-11-13T00:15:48,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742385_1561 (size=14741) 2024-11-13T00:15:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T00:15:48,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:48,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:48,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457008461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:48,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:48,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:48,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T00:15:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457008576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:48,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:48,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/79cddd5b7a4447edb4da022ffe196cfe 2024-11-13T00:15:48,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/4b37b16f1bb04899b6e9b5b44184cf05 is 50, key is test_row_0/B:col10/1731456947162/Put/seqid=0 2024-11-13T00:15:48,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742386_1562 (size=12301) 2024-11-13T00:15:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457008786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:48,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:48,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T00:15:49,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457009106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:49,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/4b37b16f1bb04899b6e9b5b44184cf05 2024-11-13T00:15:49,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/bb00a385bd3e4302b2ea16a01509928c is 50, key is test_row_0/C:col10/1731456947162/Put/seqid=0 2024-11-13T00:15:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742387_1563 (size=12301) 2024-11-13T00:15:49,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T00:15:49,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:15:49,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/bb00a385bd3e4302b2ea16a01509928c 2024-11-13T00:15:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/79cddd5b7a4447edb4da022ffe196cfe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/79cddd5b7a4447edb4da022ffe196cfe 2024-11-13T00:15:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/79cddd5b7a4447edb4da022ffe196cfe, entries=200, sequenceid=367, filesize=14.4 K 2024-11-13T00:15:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/4b37b16f1bb04899b6e9b5b44184cf05 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4b37b16f1bb04899b6e9b5b44184cf05 2024-11-13T00:15:49,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4b37b16f1bb04899b6e9b5b44184cf05, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T00:15:49,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/bb00a385bd3e4302b2ea16a01509928c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bb00a385bd3e4302b2ea16a01509928c 2024-11-13T00:15:49,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bb00a385bd3e4302b2ea16a01509928c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T00:15:49,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a66cd1bb022d966edc952b066b868a43 in 1247ms, sequenceid=367, compaction requested=true 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:49,576 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:49,576 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:49,577 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:49,577 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:49,577 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:49,577 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:49,577 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,577 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,577 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/e07b899f0f6645f9957614a26153cfba, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6dffc6f6c6aa40a8911f4e93cd9ec904, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4b37b16f1bb04899b6e9b5b44184cf05] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.7 K 2024-11-13T00:15:49,577 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8740325c96a487d98e73b265714319a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/eb38c8fc4ce641e38de7152ca62bf23d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/79cddd5b7a4447edb4da022ffe196cfe] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=39.1 K 2024-11-13T00:15:49,578 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting e07b899f0f6645f9957614a26153cfba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731456945236 2024-11-13T00:15:49,578 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8740325c96a487d98e73b265714319a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731456945236 2024-11-13T00:15:49,578 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dffc6f6c6aa40a8911f4e93cd9ec904, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731456945859 2024-11-13T00:15:49,578 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb38c8fc4ce641e38de7152ca62bf23d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731456945859 2024-11-13T00:15:49,578 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b37b16f1bb04899b6e9b5b44184cf05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456947147 2024-11-13T00:15:49,578 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79cddd5b7a4447edb4da022ffe196cfe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456947093 2024-11-13T00:15:49,584 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:49,584 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/6305569515ba4ac6bfe4aef239b78b34 is 50, key is test_row_0/B:col10/1731456947162/Put/seqid=0 2024-11-13T00:15:49,586 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:49,587 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8245309cabdd4bb1b86ae305ea9c65e2 is 50, key is test_row_0/A:col10/1731456947162/Put/seqid=0 2024-11-13T00:15:49,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742388_1564 (size=13051) 2024-11-13T00:15:49,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742389_1565 (size=13051) 2024-11-13T00:15:49,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T00:15:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,618 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:15:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:49,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:49,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:49,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c8a6beea42204cbe96147cf690d083e9 is 50, key is test_row_0/A:col10/1731456948459/Put/seqid=0 2024-11-13T00:15:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:49,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742390_1566 (size=12301) 2024-11-13T00:15:49,630 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c8a6beea42204cbe96147cf690d083e9 2024-11-13T00:15:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/7d80250e8ab3477d84a588f1a7f33224 is 50, key is test_row_0/B:col10/1731456948459/Put/seqid=0 2024-11-13T00:15:49,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742391_1567 (size=12301) 2024-11-13T00:15:49,640 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/7d80250e8ab3477d84a588f1a7f33224 2024-11-13T00:15:49,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/1f3d5bfed5d742719ce3685f87ee3a30 is 50, key is test_row_0/C:col10/1731456948459/Put/seqid=0 2024-11-13T00:15:49,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742392_1568 (size=12301) 2024-11-13T00:15:49,650 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/1f3d5bfed5d742719ce3685f87ee3a30 2024-11-13T00:15:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/c8a6beea42204cbe96147cf690d083e9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8a6beea42204cbe96147cf690d083e9 2024-11-13T00:15:49,656 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8a6beea42204cbe96147cf690d083e9, entries=150, sequenceid=387, filesize=12.0 K 2024-11-13T00:15:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/7d80250e8ab3477d84a588f1a7f33224 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7d80250e8ab3477d84a588f1a7f33224 2024-11-13T00:15:49,660 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7d80250e8ab3477d84a588f1a7f33224, entries=150, sequenceid=387, filesize=12.0 K 2024-11-13T00:15:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/1f3d5bfed5d742719ce3685f87ee3a30 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/1f3d5bfed5d742719ce3685f87ee3a30 2024-11-13T00:15:49,664 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/1f3d5bfed5d742719ce3685f87ee3a30, entries=150, sequenceid=387, filesize=12.0 K 2024-11-13T00:15:49,665 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for a66cd1bb022d966edc952b066b868a43 in 46ms, sequenceid=387, compaction requested=true 2024-11-13T00:15:49,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:49,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:49,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-13T00:15:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-13T00:15:49,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T00:15:49,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-11-13T00:15:49,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4330 sec 2024-11-13T00:15:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:49,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:15:49,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:49,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:49,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:49,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:49,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:49,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:49,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/efc731ca4f794276b35c040123d8cae9 is 50, key is test_row_0/A:col10/1731456949666/Put/seqid=0 2024-11-13T00:15:49,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742393_1569 (size=12301) 2024-11-13T00:15:49,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/efc731ca4f794276b35c040123d8cae9 2024-11-13T00:15:49,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/b16240c0683d491aa30907b1140b4f4c is 50, key is test_row_0/B:col10/1731456949666/Put/seqid=0 2024-11-13T00:15:49,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742394_1570 (size=12301) 2024-11-13T00:15:49,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456949804, max timestamp=1731456949852Num uniques 859; 2024-11-13T00:15:49,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:49,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456949804, max timestamp=1731456949852Num uniques 859; 2024-11-13T00:15:49,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:49,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456949804, max timestamp=1731456949852Num uniques 859; 2024-11-13T00:15:49,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:15:49,998 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/6305569515ba4ac6bfe4aef239b78b34 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6305569515ba4ac6bfe4aef239b78b34 2024-11-13T00:15:50,002 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into 6305569515ba4ac6bfe4aef239b78b34(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:50,002 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:50,002 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456949576; duration=0sec 2024-11-13T00:15:50,002 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:50,002 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:50,002 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:50,003 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:50,003 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:50,003 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:50,004 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/dc8720697c81473aaf81ba6f8e4c21b0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/0b4be2f6fcb7456ead2bba9d07e1a4e9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bb00a385bd3e4302b2ea16a01509928c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/1f3d5bfed5d742719ce3685f87ee3a30] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=48.7 K 2024-11-13T00:15:50,004 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8720697c81473aaf81ba6f8e4c21b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731456945236 2024-11-13T00:15:50,004 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/8245309cabdd4bb1b86ae305ea9c65e2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8245309cabdd4bb1b86ae305ea9c65e2 2024-11-13T00:15:50,004 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b4be2f6fcb7456ead2bba9d07e1a4e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731456945859 2024-11-13T00:15:50,005 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting bb00a385bd3e4302b2ea16a01509928c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456947147 2024-11-13T00:15:50,005 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3d5bfed5d742719ce3685f87ee3a30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731456948442 2024-11-13T00:15:50,008 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into 8245309cabdd4bb1b86ae305ea9c65e2(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:50,008 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:50,008 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456949576; duration=0sec 2024-11-13T00:15:50,008 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:50,008 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:50,013 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:50,013 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4edce023dee84b18a64041ff86875c08 is 50, key is test_row_0/C:col10/1731456948459/Put/seqid=0 2024-11-13T00:15:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742395_1571 (size=13085) 2024-11-13T00:15:50,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457010036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:50,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/b16240c0683d491aa30907b1140b4f4c 2024-11-13T00:15:50,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/5617fe2f53fe492a8eda56026ff35e09 is 50, key is test_row_0/C:col10/1731456949666/Put/seqid=0 2024-11-13T00:15:50,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742396_1572 (size=12301) 2024-11-13T00:15:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457010146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457010366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T00:15:50,386 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:50,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:15:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-13T00:15:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T00:15:50,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:15:50,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:15:50,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:15:50,421 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/4edce023dee84b18a64041ff86875c08 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4edce023dee84b18a64041ff86875c08 2024-11-13T00:15:50,425 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into 4edce023dee84b18a64041ff86875c08(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:50,425 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:50,425 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=12, startTime=1731456949576; duration=0sec 2024-11-13T00:15:50,425 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:50,425 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T00:15:50,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/5617fe2f53fe492a8eda56026ff35e09 2024-11-13T00:15:50,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/efc731ca4f794276b35c040123d8cae9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/efc731ca4f794276b35c040123d8cae9 2024-11-13T00:15:50,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/efc731ca4f794276b35c040123d8cae9, entries=150, sequenceid=397, filesize=12.0 K 2024-11-13T00:15:50,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/b16240c0683d491aa30907b1140b4f4c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b16240c0683d491aa30907b1140b4f4c 2024-11-13T00:15:50,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b16240c0683d491aa30907b1140b4f4c, entries=150, sequenceid=397, filesize=12.0 K 2024-11-13T00:15:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/5617fe2f53fe492a8eda56026ff35e09 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5617fe2f53fe492a8eda56026ff35e09 2024-11-13T00:15:50,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5617fe2f53fe492a8eda56026ff35e09, entries=150, sequenceid=397, filesize=12.0 K 2024-11-13T00:15:50,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a66cd1bb022d966edc952b066b868a43 in 848ms, sequenceid=397, compaction requested=true 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:50,524 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:50,524 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:50,525 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:50,525 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:50,525 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:50,525 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:50,525 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:50,525 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:50,525 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8245309cabdd4bb1b86ae305ea9c65e2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8a6beea42204cbe96147cf690d083e9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/efc731ca4f794276b35c040123d8cae9] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.8 K 2024-11-13T00:15:50,525 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6305569515ba4ac6bfe4aef239b78b34, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7d80250e8ab3477d84a588f1a7f33224, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b16240c0683d491aa30907b1140b4f4c] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.8 K 2024-11-13T00:15:50,526 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8245309cabdd4bb1b86ae305ea9c65e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456947147 2024-11-13T00:15:50,526 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6305569515ba4ac6bfe4aef239b78b34, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731456947147 2024-11-13T00:15:50,526 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8a6beea42204cbe96147cf690d083e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731456948442 2024-11-13T00:15:50,526 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d80250e8ab3477d84a588f1a7f33224, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731456948442 2024-11-13T00:15:50,526 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc731ca4f794276b35c040123d8cae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731456949644 2024-11-13T00:15:50,526 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting b16240c0683d491aa30907b1140b4f4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731456949644 2024-11-13T00:15:50,532 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#486 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:50,532 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/4c16f1f78f0b486fb05d29ee8368cc0d is 50, key is test_row_0/B:col10/1731456949666/Put/seqid=0 2024-11-13T00:15:50,532 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:50,533 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/50a989492c3b42ba9511a4d95b3ecedb is 50, key is test_row_0/A:col10/1731456949666/Put/seqid=0 2024-11-13T00:15:50,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742397_1573 (size=13153) 2024-11-13T00:15:50,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742398_1574 (size=13153) 2024-11-13T00:15:50,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:50,542 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:50,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/e30183589e974a4db0f80aee572c1cba is 50, key is test_row_0/A:col10/1731456950023/Put/seqid=0 2024-11-13T00:15:50,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742399_1575 (size=12301) 2024-11-13T00:15:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:50,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. as already flushing 2024-11-13T00:15:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T00:15:50,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457010732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:50,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457010846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:50,939 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/4c16f1f78f0b486fb05d29ee8368cc0d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4c16f1f78f0b486fb05d29ee8368cc0d 2024-11-13T00:15:50,940 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/50a989492c3b42ba9511a4d95b3ecedb as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/50a989492c3b42ba9511a4d95b3ecedb 2024-11-13T00:15:50,944 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into 4c16f1f78f0b486fb05d29ee8368cc0d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:50,944 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:50,944 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456950524; duration=0sec 2024-11-13T00:15:50,944 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:50,944 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:50,944 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T00:15:50,945 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into 50a989492c3b42ba9511a4d95b3ecedb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:50,945 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456950524; duration=0sec 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. because compaction request was cancelled 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:50,945 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:50,949 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/e30183589e974a4db0f80aee572c1cba 2024-11-13T00:15:50,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/0cfc2dbe457342c3853c4a33a446d61f is 50, key is test_row_0/B:col10/1731456950023/Put/seqid=0 2024-11-13T00:15:50,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742400_1576 (size=12301) 2024-11-13T00:15:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T00:15:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457011065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:51,360 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/0cfc2dbe457342c3853c4a33a446d61f 2024-11-13T00:15:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d78bca7c76b24ccba9c604a9c3f6ff87 is 50, key is test_row_0/C:col10/1731456950023/Put/seqid=0 2024-11-13T00:15:51,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742401_1577 (size=12301) 2024-11-13T00:15:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457011376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T00:15:51,773 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d78bca7c76b24ccba9c604a9c3f6ff87 2024-11-13T00:15:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/e30183589e974a4db0f80aee572c1cba as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e30183589e974a4db0f80aee572c1cba 2024-11-13T00:15:51,780 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e30183589e974a4db0f80aee572c1cba, entries=150, sequenceid=436, filesize=12.0 K 2024-11-13T00:15:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/0cfc2dbe457342c3853c4a33a446d61f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0cfc2dbe457342c3853c4a33a446d61f 2024-11-13T00:15:51,784 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0cfc2dbe457342c3853c4a33a446d61f, entries=150, sequenceid=436, filesize=12.0 K 2024-11-13T00:15:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/d78bca7c76b24ccba9c604a9c3f6ff87 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d78bca7c76b24ccba9c604a9c3f6ff87 2024-11-13T00:15:51,787 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d78bca7c76b24ccba9c604a9c3f6ff87, entries=150, sequenceid=436, filesize=12.0 K 2024-11-13T00:15:51,788 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a66cd1bb022d966edc952b066b868a43 in 1246ms, sequenceid=436, compaction requested=true 2024-11-13T00:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-13T00:15:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-13T00:15:51,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-13T00:15:51,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-13T00:15:51,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4050 sec 2024-11-13T00:15:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:51,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:15:51,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:51,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:51,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:51,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:51,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:51,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:51,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b738614a84ef4ffdbf5f1823262413c1 is 50, key is test_row_0/A:col10/1731456951897/Put/seqid=0 2024-11-13T00:15:51,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742402_1578 (size=14741) 2024-11-13T00:15:52,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456951983, max timestamp=1731456952019Num uniques 859; 2024-11-13T00:15:52,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:52,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456951983, max timestamp=1731456952019Num uniques 859; 2024-11-13T00:15:52,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:52,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456951983, max timestamp=1731456952019Num uniques 859; 2024-11-13T00:15:52,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:15:52,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457012149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457012266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:52,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b738614a84ef4ffdbf5f1823262413c1 2024-11-13T00:15:52,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/fbced355a06d4a7f8703ce3d58ddf409 is 50, key is test_row_0/B:col10/1731456951897/Put/seqid=0 2024-11-13T00:15:52,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742403_1579 (size=12301) 2024-11-13T00:15:52,321 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-13T00:15:52,321 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:52,321 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,321 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:52,324 INFO [Thread-2224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2224. 2024-11-13T00:15:52,324 DEBUG [Thread-2224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:52,324 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,324 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:52,331 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-13T00:15:52,331 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:52,331 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,331 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:52,337 INFO [Thread-2222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2222. 2024-11-13T00:15:52,337 DEBUG [Thread-2222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:52,337 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,338 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:52,339 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-11-13T00:15:52,339 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:52,339 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,339 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:52,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:52,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457012486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T00:15:52,536 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:52,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/fbced355a06d4a7f8703ce3d58ddf409 2024-11-13T00:15:52,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/a2408ab445864a35b59d2bfbcc9c69ef is 50, key is test_row_0/C:col10/1731456951897/Put/seqid=0 2024-11-13T00:15:52,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742404_1580 (size=12301) 2024-11-13T00:15:52,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59228 deadline: 1731457012796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 2024-11-13T00:15:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 , the old value is region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:15:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., hostname=cb28580903fb,45273,1731456804594, seqNum=2 because the exception is null or not the one we care about 2024-11-13T00:15:52,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:15:53,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/a2408ab445864a35b59d2bfbcc9c69ef 2024-11-13T00:15:53,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/b738614a84ef4ffdbf5f1823262413c1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b738614a84ef4ffdbf5f1823262413c1 2024-11-13T00:15:53,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b738614a84ef4ffdbf5f1823262413c1, entries=200, sequenceid=449, filesize=14.4 K 2024-11-13T00:15:53,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/fbced355a06d4a7f8703ce3d58ddf409 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/fbced355a06d4a7f8703ce3d58ddf409 2024-11-13T00:15:53,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/fbced355a06d4a7f8703ce3d58ddf409, entries=150, sequenceid=449, filesize=12.0 K 2024-11-13T00:15:53,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/a2408ab445864a35b59d2bfbcc9c69ef as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a2408ab445864a35b59d2bfbcc9c69ef 2024-11-13T00:15:53,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a2408ab445864a35b59d2bfbcc9c69ef, entries=150, sequenceid=449, filesize=12.0 K 2024-11-13T00:15:53,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a66cd1bb022d966edc952b066b868a43 in 1243ms, sequenceid=449, compaction requested=true 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a66cd1bb022d966edc952b066b868a43:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:15:53,142 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:53,142 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:53,143 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:53,143 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/A is initiating minor compaction (all files) 2024-11-13T00:15:53,143 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/A in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:53,143 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/50a989492c3b42ba9511a4d95b3ecedb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e30183589e974a4db0f80aee572c1cba, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b738614a84ef4ffdbf5f1823262413c1] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=39.3 K 2024-11-13T00:15:53,143 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:15:53,143 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/B is initiating minor compaction (all files) 2024-11-13T00:15:53,144 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/B in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:53,144 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4c16f1f78f0b486fb05d29ee8368cc0d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0cfc2dbe457342c3853c4a33a446d61f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/fbced355a06d4a7f8703ce3d58ddf409] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=36.9 K 2024-11-13T00:15:53,144 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a989492c3b42ba9511a4d95b3ecedb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731456949644 2024-11-13T00:15:53,144 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c16f1f78f0b486fb05d29ee8368cc0d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731456949644 2024-11-13T00:15:53,144 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting e30183589e974a4db0f80aee572c1cba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731456949970 2024-11-13T00:15:53,144 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cfc2dbe457342c3853c4a33a446d61f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731456949970 2024-11-13T00:15:53,145 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting fbced355a06d4a7f8703ce3d58ddf409, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731456950705 2024-11-13T00:15:53,145 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting b738614a84ef4ffdbf5f1823262413c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731456950705 2024-11-13T00:15:53,151 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#A#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:53,152 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/6ecd4fc9c2464ce9bd7642657c067e36 is 50, key is test_row_0/A:col10/1731456951897/Put/seqid=0 2024-11-13T00:15:53,153 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:53,154 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/0105135a797f41ffbfb90e409cb55981 is 50, key is test_row_0/B:col10/1731456951897/Put/seqid=0 2024-11-13T00:15:53,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742405_1581 (size=13255) 2024-11-13T00:15:53,164 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/6ecd4fc9c2464ce9bd7642657c067e36 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/6ecd4fc9c2464ce9bd7642657c067e36 2024-11-13T00:15:53,168 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/A of a66cd1bb022d966edc952b066b868a43 into 6ecd4fc9c2464ce9bd7642657c067e36(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:53,168 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:53,168 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/A, priority=13, startTime=1731456953142; duration=0sec 2024-11-13T00:15:53,169 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:15:53,169 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:A 2024-11-13T00:15:53,169 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:15:53,170 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:15:53,170 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): a66cd1bb022d966edc952b066b868a43/C is initiating minor compaction (all files) 2024-11-13T00:15:53,171 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a66cd1bb022d966edc952b066b868a43/C in TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:53,171 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4edce023dee84b18a64041ff86875c08, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5617fe2f53fe492a8eda56026ff35e09, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d78bca7c76b24ccba9c604a9c3f6ff87, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a2408ab445864a35b59d2bfbcc9c69ef] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp, totalSize=48.8 K 2024-11-13T00:15:53,171 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4edce023dee84b18a64041ff86875c08, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731456948442 2024-11-13T00:15:53,171 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5617fe2f53fe492a8eda56026ff35e09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731456949644 2024-11-13T00:15:53,172 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78bca7c76b24ccba9c604a9c3f6ff87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731456949970 2024-11-13T00:15:53,172 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2408ab445864a35b59d2bfbcc9c69ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731456950705 2024-11-13T00:15:53,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742406_1582 (size=13255) 2024-11-13T00:15:53,195 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a66cd1bb022d966edc952b066b868a43#C#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:15:53,196 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/330ff12308b1478e8c33c27693d5f527 is 50, key is test_row_0/C:col10/1731456951897/Put/seqid=0 2024-11-13T00:15:53,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742407_1583 (size=13221) 2024-11-13T00:15:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:53,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T00:15:53,308 INFO [Thread-2211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2211. 2024-11-13T00:15:53,308 DEBUG [Thread-2211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:53,308 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:53,308 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:53,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:15:53,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/cbd492ba389f4539b857cb745091bd0c is 50, key is test_row_0/A:col10/1731456952095/Put/seqid=0 2024-11-13T00:15:53,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742408_1584 (size=12301) 2024-11-13T00:15:53,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/cbd492ba389f4539b857cb745091bd0c 2024-11-13T00:15:53,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/55b0152c46c047709e7d3c0d5ebb9972 is 50, key is test_row_0/B:col10/1731456952095/Put/seqid=0 2024-11-13T00:15:53,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742409_1585 (size=12301) 2024-11-13T00:15:53,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/55b0152c46c047709e7d3c0d5ebb9972 2024-11-13T00:15:53,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/3a837fc1b42e4b618e27cce9b51a780a is 50, key is test_row_0/C:col10/1731456952095/Put/seqid=0 2024-11-13T00:15:53,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742410_1586 (size=12301) 2024-11-13T00:15:53,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/3a837fc1b42e4b618e27cce9b51a780a 2024-11-13T00:15:53,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/cbd492ba389f4539b857cb745091bd0c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/cbd492ba389f4539b857cb745091bd0c 2024-11-13T00:15:53,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/cbd492ba389f4539b857cb745091bd0c, entries=150, sequenceid=487, filesize=12.0 K 2024-11-13T00:15:53,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/55b0152c46c047709e7d3c0d5ebb9972 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/55b0152c46c047709e7d3c0d5ebb9972 2024-11-13T00:15:53,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/55b0152c46c047709e7d3c0d5ebb9972, entries=150, sequenceid=487, filesize=12.0 K 2024-11-13T00:15:53,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/3a837fc1b42e4b618e27cce9b51a780a as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3a837fc1b42e4b618e27cce9b51a780a 2024-11-13T00:15:53,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3a837fc1b42e4b618e27cce9b51a780a, entries=150, sequenceid=487, filesize=12.0 K 2024-11-13T00:15:53,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for a66cd1bb022d966edc952b066b868a43 in 74ms, sequenceid=487, compaction requested=false 2024-11-13T00:15:53,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:53,579 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/0105135a797f41ffbfb90e409cb55981 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0105135a797f41ffbfb90e409cb55981 2024-11-13T00:15:53,583 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a66cd1bb022d966edc952b066b868a43/B of a66cd1bb022d966edc952b066b868a43 into 0105135a797f41ffbfb90e409cb55981(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:53,583 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:53,583 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/B, priority=13, startTime=1731456953142; duration=0sec 2024-11-13T00:15:53,583 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:53,583 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:B 2024-11-13T00:15:53,603 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/330ff12308b1478e8c33c27693d5f527 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/330ff12308b1478e8c33c27693d5f527 2024-11-13T00:15:53,607 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a66cd1bb022d966edc952b066b868a43/C of a66cd1bb022d966edc952b066b868a43 into 330ff12308b1478e8c33c27693d5f527(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:15:53,607 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a66cd1bb022d966edc952b066b868a43: 2024-11-13T00:15:53,607 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., storeName=a66cd1bb022d966edc952b066b868a43/C, priority=12, startTime=1731456953142; duration=0sec 2024-11-13T00:15:53,607 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:15:53,607 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a66cd1bb022d966edc952b066b868a43:C 2024-11-13T00:15:57,347 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-13T00:15:57,347 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:57,347 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,347 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-11-13T00:15:57,347 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,347 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:57,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:57,347 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,347 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:57,408 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-13T00:15:57,408 INFO [Thread-2213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2213. 2024-11-13T00:15:57,408 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:57,408 DEBUG [Thread-2213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:15:57,408 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,408 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,408 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,408 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:57,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:57,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T00:15:57,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-13T00:15:57,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-13T00:15:57,408 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 269 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1063 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3187 rows 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3169 rows 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1056 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3168 rows 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3173 rows 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1055 2024-11-13T00:15:57,409 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3165 rows 2024-11-13T00:15:57,409 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:15:57,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:15:57,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:15:57,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:15:57,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T00:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T00:15:57,412 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456957412"}]},"ts":"1731456957412"} 2024-11-13T00:15:57,413 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T00:15:57,413 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T00:15:57,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:15:57,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, UNASSIGN}] 2024-11-13T00:15:57,415 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, UNASSIGN 2024-11-13T00:15:57,416 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a66cd1bb022d966edc952b066b868a43, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:15:57,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, UNASSIGN because future has completed 2024-11-13T00:15:57,417 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:15:57,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T00:15:57,569 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing a66cd1bb022d966edc952b066b868a43, disabling compactions & flushes 2024-11-13T00:15:57,569 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. after waiting 0 ms 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:57,569 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing a66cd1bb022d966edc952b066b868a43 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=A 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=B 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a66cd1bb022d966edc952b066b868a43, store=C 2024-11-13T00:15:57,569 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:15:57,572 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/bf7f20d17370485da40e16b87a7dc6dd is 50, key is test_row_2/A:col10/1731456957407/Put/seqid=0 2024-11-13T00:15:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742411_1587 (size=7415) 2024-11-13T00:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T00:15:57,976 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/bf7f20d17370485da40e16b87a7dc6dd 2024-11-13T00:15:57,981 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/412aafe186a144c493780cb7409edcb3 is 50, key is test_row_2/B:col10/1731456957407/Put/seqid=0 2024-11-13T00:15:57,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742412_1588 (size=7415) 2024-11-13T00:15:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T00:15:58,385 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/412aafe186a144c493780cb7409edcb3 2024-11-13T00:15:58,390 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/2ac8bf35c15a4e4499894aac2eec7e94 is 50, key is test_row_2/C:col10/1731456957407/Put/seqid=0 2024-11-13T00:15:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742413_1589 (size=7415) 2024-11-13T00:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T00:15:58,793 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/2ac8bf35c15a4e4499894aac2eec7e94 2024-11-13T00:15:58,797 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/A/bf7f20d17370485da40e16b87a7dc6dd as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bf7f20d17370485da40e16b87a7dc6dd 2024-11-13T00:15:58,799 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bf7f20d17370485da40e16b87a7dc6dd, entries=50, sequenceid=496, filesize=7.2 K 2024-11-13T00:15:58,800 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/B/412aafe186a144c493780cb7409edcb3 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/412aafe186a144c493780cb7409edcb3 2024-11-13T00:15:58,802 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/412aafe186a144c493780cb7409edcb3, entries=50, sequenceid=496, filesize=7.2 K 2024-11-13T00:15:58,803 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/.tmp/C/2ac8bf35c15a4e4499894aac2eec7e94 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/2ac8bf35c15a4e4499894aac2eec7e94 2024-11-13T00:15:58,805 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/2ac8bf35c15a4e4499894aac2eec7e94, entries=50, sequenceid=496, filesize=7.2 K 2024-11-13T00:15:58,806 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a66cd1bb022d966edc952b066b868a43 in 1237ms, sequenceid=496, compaction requested=true 2024-11-13T00:15:58,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/72c57024559f450fa31afd2fb38d202a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fa98abe6d43b4d5e9e16a327f7e8572e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0622314387c74ccdabd8ab8f8f15ad21, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ad366221e1fe4d30b7df74449be988ae, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b2845232dff6448e9c2eb2cc95bd7d11, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/51be1f92211c49569d429ab0b4bb5f2a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b8bf801b91844f96b99b68358ccd0700, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ea70dab339ec4e29b0439b962e301075, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8e530db524b84d57a6f2f38754729f3c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8b410fd98de946ae8809c5ab757bca61, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c5e3628547db4ca6b81b0bd8da1e6fd9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bb8653662aea4cdbad448204d712594c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ff8226046a59410da5963d917a1e1bd7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/1eda4151c1e1416685ca4027de16ff40, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/5b0c745d3df44c189a75ff06efa2a8ab, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/a5e63583e91043c18b6f59d6a4751b81, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0e0572939d3242b3b94a803272102293, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fac91fef06804273ada092403671ca27, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8740325c96a487d98e73b265714319a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/eb38c8fc4ce641e38de7152ca62bf23d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/79cddd5b7a4447edb4da022ffe196cfe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8245309cabdd4bb1b86ae305ea9c65e2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8a6beea42204cbe96147cf690d083e9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/50a989492c3b42ba9511a4d95b3ecedb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/efc731ca4f794276b35c040123d8cae9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e30183589e974a4db0f80aee572c1cba, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b738614a84ef4ffdbf5f1823262413c1] to archive 2024-11-13T00:15:58,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:58,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/72c57024559f450fa31afd2fb38d202a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/72c57024559f450fa31afd2fb38d202a 2024-11-13T00:15:58,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fa98abe6d43b4d5e9e16a327f7e8572e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fa98abe6d43b4d5e9e16a327f7e8572e 2024-11-13T00:15:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0622314387c74ccdabd8ab8f8f15ad21 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0622314387c74ccdabd8ab8f8f15ad21 2024-11-13T00:15:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ad366221e1fe4d30b7df74449be988ae to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ad366221e1fe4d30b7df74449be988ae 2024-11-13T00:15:58,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b2845232dff6448e9c2eb2cc95bd7d11 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b2845232dff6448e9c2eb2cc95bd7d11 2024-11-13T00:15:58,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e2a550103e1b4dcf9c49fa7b4a2ef0bc 2024-11-13T00:15:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/51be1f92211c49569d429ab0b4bb5f2a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/51be1f92211c49569d429ab0b4bb5f2a 2024-11-13T00:15:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b8bf801b91844f96b99b68358ccd0700 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b8bf801b91844f96b99b68358ccd0700 2024-11-13T00:15:58,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ea70dab339ec4e29b0439b962e301075 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ea70dab339ec4e29b0439b962e301075 2024-11-13T00:15:58,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8e530db524b84d57a6f2f38754729f3c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8e530db524b84d57a6f2f38754729f3c 2024-11-13T00:15:58,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8b410fd98de946ae8809c5ab757bca61 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8b410fd98de946ae8809c5ab757bca61 2024-11-13T00:15:58,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c5e3628547db4ca6b81b0bd8da1e6fd9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c5e3628547db4ca6b81b0bd8da1e6fd9 2024-11-13T00:15:58,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bb8653662aea4cdbad448204d712594c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bb8653662aea4cdbad448204d712594c 2024-11-13T00:15:58,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ff8226046a59410da5963d917a1e1bd7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/ff8226046a59410da5963d917a1e1bd7 2024-11-13T00:15:58,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/1eda4151c1e1416685ca4027de16ff40 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/1eda4151c1e1416685ca4027de16ff40 2024-11-13T00:15:58,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/5b0c745d3df44c189a75ff06efa2a8ab to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/5b0c745d3df44c189a75ff06efa2a8ab 2024-11-13T00:15:58,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/a5e63583e91043c18b6f59d6a4751b81 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/a5e63583e91043c18b6f59d6a4751b81 2024-11-13T00:15:58,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0e0572939d3242b3b94a803272102293 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/0e0572939d3242b3b94a803272102293 2024-11-13T00:15:58,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fac91fef06804273ada092403671ca27 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/fac91fef06804273ada092403671ca27 2024-11-13T00:15:58,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8740325c96a487d98e73b265714319a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8740325c96a487d98e73b265714319a 2024-11-13T00:15:58,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/eb38c8fc4ce641e38de7152ca62bf23d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/eb38c8fc4ce641e38de7152ca62bf23d 2024-11-13T00:15:58,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/79cddd5b7a4447edb4da022ffe196cfe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/79cddd5b7a4447edb4da022ffe196cfe 2024-11-13T00:15:58,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8245309cabdd4bb1b86ae305ea9c65e2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/8245309cabdd4bb1b86ae305ea9c65e2 2024-11-13T00:15:58,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8a6beea42204cbe96147cf690d083e9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/c8a6beea42204cbe96147cf690d083e9 2024-11-13T00:15:58,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/50a989492c3b42ba9511a4d95b3ecedb to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/50a989492c3b42ba9511a4d95b3ecedb 2024-11-13T00:15:58,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/efc731ca4f794276b35c040123d8cae9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/efc731ca4f794276b35c040123d8cae9 2024-11-13T00:15:58,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e30183589e974a4db0f80aee572c1cba to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/e30183589e974a4db0f80aee572c1cba 2024-11-13T00:15:58,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b738614a84ef4ffdbf5f1823262413c1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/b738614a84ef4ffdbf5f1823262413c1 2024-11-13T00:15:58,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a520a7f733774332ad14545df83a7d7a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f9ec44bdb2f34343b4a29ac371c30686, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a52fda3de44d418bab29929fbfc2449a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/d69b7f91b7ed45a58d90655adf9b0412, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/bf0a95dd7db44382a7ebe155b984f782, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b843e714cf694a92b26df4b2254be88f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/ac19545098614c93bf66fd086503353b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/097129155f9a4fcb9e4819a0a2764564, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/8effb56f501b4f02a75620e69cc91ff4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7b359ba29cb44d3e9713598c91cf0711, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/29089fbd1ec54efa8a7f2a3b1be5b0ff, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/9d9efe7f923d448abf022e5ecf034630, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/60a7cd4b7e224cbda5410f43612b6f7a, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/c9f6caf7fbb2454ebad63cad6cbd5159, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/216939ce6cc4462689077fd762e00acc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/321e9a1d37364be98e0fbe2b62958862, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f3110799eaea4488b69218e489213d40, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/03144b76aa4d4935936553d76922cbbc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/e07b899f0f6645f9957614a26153cfba, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f7222acb708748f7928b24bf9b98a6b6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6dffc6f6c6aa40a8911f4e93cd9ec904, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6305569515ba4ac6bfe4aef239b78b34, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4b37b16f1bb04899b6e9b5b44184cf05, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7d80250e8ab3477d84a588f1a7f33224, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4c16f1f78f0b486fb05d29ee8368cc0d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b16240c0683d491aa30907b1140b4f4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0cfc2dbe457342c3853c4a33a446d61f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/fbced355a06d4a7f8703ce3d58ddf409] to archive 2024-11-13T00:15:58,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:58,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a520a7f733774332ad14545df83a7d7a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a520a7f733774332ad14545df83a7d7a 2024-11-13T00:15:58,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f9ec44bdb2f34343b4a29ac371c30686 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f9ec44bdb2f34343b4a29ac371c30686 2024-11-13T00:15:58,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a52fda3de44d418bab29929fbfc2449a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/a52fda3de44d418bab29929fbfc2449a 2024-11-13T00:15:58,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/d69b7f91b7ed45a58d90655adf9b0412 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/d69b7f91b7ed45a58d90655adf9b0412 2024-11-13T00:15:58,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/bf0a95dd7db44382a7ebe155b984f782 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/bf0a95dd7db44382a7ebe155b984f782 2024-11-13T00:15:58,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b843e714cf694a92b26df4b2254be88f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b843e714cf694a92b26df4b2254be88f 2024-11-13T00:15:58,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/ac19545098614c93bf66fd086503353b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/ac19545098614c93bf66fd086503353b 2024-11-13T00:15:58,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/097129155f9a4fcb9e4819a0a2764564 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/097129155f9a4fcb9e4819a0a2764564 2024-11-13T00:15:58,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/8effb56f501b4f02a75620e69cc91ff4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/8effb56f501b4f02a75620e69cc91ff4 2024-11-13T00:15:58,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7b359ba29cb44d3e9713598c91cf0711 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7b359ba29cb44d3e9713598c91cf0711 2024-11-13T00:15:58,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/29089fbd1ec54efa8a7f2a3b1be5b0ff to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/29089fbd1ec54efa8a7f2a3b1be5b0ff 2024-11-13T00:15:58,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/9d9efe7f923d448abf022e5ecf034630 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/9d9efe7f923d448abf022e5ecf034630 2024-11-13T00:15:58,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/60a7cd4b7e224cbda5410f43612b6f7a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/60a7cd4b7e224cbda5410f43612b6f7a 2024-11-13T00:15:58,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/c9f6caf7fbb2454ebad63cad6cbd5159 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/c9f6caf7fbb2454ebad63cad6cbd5159 2024-11-13T00:15:58,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/216939ce6cc4462689077fd762e00acc to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/216939ce6cc4462689077fd762e00acc 2024-11-13T00:15:58,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/321e9a1d37364be98e0fbe2b62958862 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/321e9a1d37364be98e0fbe2b62958862 2024-11-13T00:15:58,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f3110799eaea4488b69218e489213d40 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f3110799eaea4488b69218e489213d40 2024-11-13T00:15:58,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/03144b76aa4d4935936553d76922cbbc to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/03144b76aa4d4935936553d76922cbbc 2024-11-13T00:15:58,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/e07b899f0f6645f9957614a26153cfba to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/e07b899f0f6645f9957614a26153cfba 2024-11-13T00:15:58,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f7222acb708748f7928b24bf9b98a6b6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/f7222acb708748f7928b24bf9b98a6b6 2024-11-13T00:15:58,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6dffc6f6c6aa40a8911f4e93cd9ec904 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6dffc6f6c6aa40a8911f4e93cd9ec904 2024-11-13T00:15:58,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6305569515ba4ac6bfe4aef239b78b34 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/6305569515ba4ac6bfe4aef239b78b34 2024-11-13T00:15:58,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4b37b16f1bb04899b6e9b5b44184cf05 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4b37b16f1bb04899b6e9b5b44184cf05 2024-11-13T00:15:58,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7d80250e8ab3477d84a588f1a7f33224 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/7d80250e8ab3477d84a588f1a7f33224 2024-11-13T00:15:58,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4c16f1f78f0b486fb05d29ee8368cc0d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/4c16f1f78f0b486fb05d29ee8368cc0d 2024-11-13T00:15:58,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b16240c0683d491aa30907b1140b4f4c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/b16240c0683d491aa30907b1140b4f4c 2024-11-13T00:15:58,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0cfc2dbe457342c3853c4a33a446d61f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0cfc2dbe457342c3853c4a33a446d61f 2024-11-13T00:15:58,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/fbced355a06d4a7f8703ce3d58ddf409 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/fbced355a06d4a7f8703ce3d58ddf409 2024-11-13T00:15:58,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f5ad3a3f34414cfabbe8005ebce16137, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/325d13bb83754e5caa7a94642fcc06ab, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e00f75a4e3214c68a870d3d56e233b2e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/728cae57a56b4b749e6b904554034063, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d5a609f630ef49738a29fa80e050d53b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4a5d2aaf60be4c81809e07e1986a203f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e1587781d39a4af2b5e7d1fce2eacae7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/fc45f480e7234501a8e5246eddf6eda3, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/145007e812c54d0e814bb53dccdbe5c5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/84440b50f617467ab9438d7bcc79f252, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/420bfdba7db94a629a905aa2a56ee816, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/cbf66ceee5f141609bc5681b644e4b84, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bd7ca74381fe4e15b0290b0899c51cd7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4eba524c432c486da6b6e7ee8c937b58, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3896ad3862b84407a19c471877d84be7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d3770d7faaa742a192da7f117f769f10, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a3ac57c8a6b14e019a1279a74cefe4e6, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f8e484ccd824416e8c335b9c9dc2d465, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/dc8720697c81473aaf81ba6f8e4c21b0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5dfbd1aa6e1b49f0b6f503647a734df1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/0b4be2f6fcb7456ead2bba9d07e1a4e9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bb00a385bd3e4302b2ea16a01509928c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4edce023dee84b18a64041ff86875c08, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/1f3d5bfed5d742719ce3685f87ee3a30, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5617fe2f53fe492a8eda56026ff35e09, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d78bca7c76b24ccba9c604a9c3f6ff87, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a2408ab445864a35b59d2bfbcc9c69ef] to archive 2024-11-13T00:15:58,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:15:58,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f5ad3a3f34414cfabbe8005ebce16137 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f5ad3a3f34414cfabbe8005ebce16137 2024-11-13T00:15:58,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/325d13bb83754e5caa7a94642fcc06ab to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/325d13bb83754e5caa7a94642fcc06ab 2024-11-13T00:15:58,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e00f75a4e3214c68a870d3d56e233b2e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e00f75a4e3214c68a870d3d56e233b2e 2024-11-13T00:15:58,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/728cae57a56b4b749e6b904554034063 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/728cae57a56b4b749e6b904554034063 2024-11-13T00:15:58,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d5a609f630ef49738a29fa80e050d53b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d5a609f630ef49738a29fa80e050d53b 2024-11-13T00:15:58,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4a5d2aaf60be4c81809e07e1986a203f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4a5d2aaf60be4c81809e07e1986a203f 2024-11-13T00:15:58,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e1587781d39a4af2b5e7d1fce2eacae7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/e1587781d39a4af2b5e7d1fce2eacae7 2024-11-13T00:15:58,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/fc45f480e7234501a8e5246eddf6eda3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/fc45f480e7234501a8e5246eddf6eda3 2024-11-13T00:15:58,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/145007e812c54d0e814bb53dccdbe5c5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/145007e812c54d0e814bb53dccdbe5c5 2024-11-13T00:15:58,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/84440b50f617467ab9438d7bcc79f252 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/84440b50f617467ab9438d7bcc79f252 2024-11-13T00:15:58,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/420bfdba7db94a629a905aa2a56ee816 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/420bfdba7db94a629a905aa2a56ee816 2024-11-13T00:15:58,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/cbf66ceee5f141609bc5681b644e4b84 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/cbf66ceee5f141609bc5681b644e4b84 2024-11-13T00:15:58,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bd7ca74381fe4e15b0290b0899c51cd7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bd7ca74381fe4e15b0290b0899c51cd7 2024-11-13T00:15:58,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4eba524c432c486da6b6e7ee8c937b58 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4eba524c432c486da6b6e7ee8c937b58 2024-11-13T00:15:58,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3896ad3862b84407a19c471877d84be7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3896ad3862b84407a19c471877d84be7 2024-11-13T00:15:58,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d3770d7faaa742a192da7f117f769f10 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d3770d7faaa742a192da7f117f769f10 2024-11-13T00:15:58,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a3ac57c8a6b14e019a1279a74cefe4e6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a3ac57c8a6b14e019a1279a74cefe4e6 2024-11-13T00:15:58,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f8e484ccd824416e8c335b9c9dc2d465 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/f8e484ccd824416e8c335b9c9dc2d465 2024-11-13T00:15:58,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/dc8720697c81473aaf81ba6f8e4c21b0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/dc8720697c81473aaf81ba6f8e4c21b0 2024-11-13T00:15:58,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5dfbd1aa6e1b49f0b6f503647a734df1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5dfbd1aa6e1b49f0b6f503647a734df1 2024-11-13T00:15:58,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/0b4be2f6fcb7456ead2bba9d07e1a4e9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/0b4be2f6fcb7456ead2bba9d07e1a4e9 2024-11-13T00:15:58,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bb00a385bd3e4302b2ea16a01509928c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/bb00a385bd3e4302b2ea16a01509928c 2024-11-13T00:15:58,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4edce023dee84b18a64041ff86875c08 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/4edce023dee84b18a64041ff86875c08 2024-11-13T00:15:58,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/1f3d5bfed5d742719ce3685f87ee3a30 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/1f3d5bfed5d742719ce3685f87ee3a30 2024-11-13T00:15:58,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5617fe2f53fe492a8eda56026ff35e09 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/5617fe2f53fe492a8eda56026ff35e09 2024-11-13T00:15:58,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d78bca7c76b24ccba9c604a9c3f6ff87 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/d78bca7c76b24ccba9c604a9c3f6ff87 2024-11-13T00:15:58,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a2408ab445864a35b59d2bfbcc9c69ef to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/a2408ab445864a35b59d2bfbcc9c69ef 2024-11-13T00:15:58,879 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/recovered.edits/499.seqid, newMaxSeqId=499, maxSeqId=1 2024-11-13T00:15:58,879 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. 2024-11-13T00:15:58,880 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for a66cd1bb022d966edc952b066b868a43: Waiting for close lock at 1731456957569Running coprocessor pre-close hooks at 1731456957569Disabling compacts and flushes for region at 1731456957569Disabling writes for close at 1731456957569Obtaining lock to block concurrent updates at 1731456957569Preparing flush snapshotting stores in a66cd1bb022d966edc952b066b868a43 at 1731456957569Finished memstore snapshotting TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731456957569Flushing stores of TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43. at 1731456957570 (+1 ms)Flushing a66cd1bb022d966edc952b066b868a43/A: creating writer at 1731456957570Flushing a66cd1bb022d966edc952b066b868a43/A: appending metadata at 1731456957572 (+2 ms)Flushing a66cd1bb022d966edc952b066b868a43/A: closing flushed file at 1731456957572Flushing a66cd1bb022d966edc952b066b868a43/B: creating writer at 1731456957979 (+407 ms)Flushing a66cd1bb022d966edc952b066b868a43/B: appending metadata at 1731456957981 (+2 ms)Flushing a66cd1bb022d966edc952b066b868a43/B: closing flushed file at 1731456957981Flushing a66cd1bb022d966edc952b066b868a43/C: creating writer at 1731456958388 (+407 ms)Flushing a66cd1bb022d966edc952b066b868a43/C: appending metadata at 1731456958390 (+2 ms)Flushing a66cd1bb022d966edc952b066b868a43/C: closing flushed file at 1731456958390Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b43827e: reopening flushed file at 1731456958796 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@639136b6: reopening flushed file at 1731456958799 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9f2fa69: reopening flushed file at 1731456958802 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a66cd1bb022d966edc952b066b868a43 in 1237ms, sequenceid=496, compaction requested=true at 1731456958806 (+4 ms)Writing region close event to WAL at 1731456958877 (+71 ms)Running coprocessor post-close hooks at 1731456958879 (+2 ms)Closed at 1731456958879 2024-11-13T00:15:58,881 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:58,882 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a66cd1bb022d966edc952b066b868a43, regionState=CLOSED 2024-11-13T00:15:58,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:15:58,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T00:15:58,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure a66cd1bb022d966edc952b066b868a43, server=cb28580903fb,45273,1731456804594 in 1.4670 sec 2024-11-13T00:15:58,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-13T00:15:58,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a66cd1bb022d966edc952b066b868a43, UNASSIGN in 1.4710 sec 2024-11-13T00:15:58,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T00:15:58,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-13T00:15:58,889 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456958889"}]},"ts":"1731456958889"} 2024-11-13T00:15:58,890 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T00:15:58,890 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T00:15:58,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-13T00:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T00:15:59,556 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:59,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T00:15:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,558 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-13T00:15:59,558 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,560 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:59,562 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/recovered.edits] 2024-11-13T00:15:59,564 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/6ecd4fc9c2464ce9bd7642657c067e36 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/6ecd4fc9c2464ce9bd7642657c067e36 2024-11-13T00:15:59,565 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bf7f20d17370485da40e16b87a7dc6dd to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/bf7f20d17370485da40e16b87a7dc6dd 2024-11-13T00:15:59,565 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/cbd492ba389f4539b857cb745091bd0c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/A/cbd492ba389f4539b857cb745091bd0c 2024-11-13T00:15:59,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0105135a797f41ffbfb90e409cb55981 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/0105135a797f41ffbfb90e409cb55981 2024-11-13T00:15:59,568 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/412aafe186a144c493780cb7409edcb3 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/412aafe186a144c493780cb7409edcb3 2024-11-13T00:15:59,569 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/55b0152c46c047709e7d3c0d5ebb9972 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/B/55b0152c46c047709e7d3c0d5ebb9972 2024-11-13T00:15:59,570 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/2ac8bf35c15a4e4499894aac2eec7e94 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/2ac8bf35c15a4e4499894aac2eec7e94 2024-11-13T00:15:59,571 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/330ff12308b1478e8c33c27693d5f527 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/330ff12308b1478e8c33c27693d5f527 2024-11-13T00:15:59,572 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3a837fc1b42e4b618e27cce9b51a780a to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/C/3a837fc1b42e4b618e27cce9b51a780a 2024-11-13T00:15:59,574 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/recovered.edits/499.seqid to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43/recovered.edits/499.seqid 2024-11-13T00:15:59,574 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/a66cd1bb022d966edc952b066b868a43 2024-11-13T00:15:59,575 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T00:15:59,576 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,578 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T00:15:59,580 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T00:15:59,581 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,581 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T00:15:59,581 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731456959581"}]},"ts":"9223372036854775807"} 2024-11-13T00:15:59,582 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T00:15:59,582 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a66cd1bb022d966edc952b066b868a43, NAME => 'TestAcidGuarantees,,1731456930125.a66cd1bb022d966edc952b066b868a43.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T00:15:59,583 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T00:15:59,583 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731456959583"}]},"ts":"9223372036854775807"} 2024-11-13T00:15:59,584 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T00:15:59,585 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-13T00:15:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-13T00:15:59,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T00:15:59,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:15:59,675 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=445 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=783 (was 844) 2024-11-13T00:15:59,683 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=783 2024-11-13T00:15:59,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:15:59,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:15:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T00:15:59,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:15:59,687 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:15:59,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-13T00:15:59,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:15:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T00:15:59,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742414_1590 (size=1061) 2024-11-13T00:15:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T00:16:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T00:16:00,093 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54 2024-11-13T00:16:00,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742415_1591 (size=53) 2024-11-13T00:16:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T00:16:00,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:16:00,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 041ae204889b5285cb06226a89ff8fa1, disabling compactions & flushes 2024-11-13T00:16:00,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. after waiting 0 ms 2024-11-13T00:16:00,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 041ae204889b5285cb06226a89ff8fa1: Waiting for close lock at 1731456960498Disabling compacts and flushes for region at 1731456960498Disabling writes for close at 1731456960498Writing region close event to WAL at 1731456960498Closed at 1731456960498 2024-11-13T00:16:00,499 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:16:00,500 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731456960499"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731456960499"}]},"ts":"1731456960499"} 2024-11-13T00:16:00,501 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:16:00,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:16:00,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456960502"}]},"ts":"1731456960502"} 2024-11-13T00:16:00,503 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T00:16:00,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, ASSIGN}] 2024-11-13T00:16:00,504 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, ASSIGN 2024-11-13T00:16:00,504 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, ASSIGN; state=OFFLINE, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=false 2024-11-13T00:16:00,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:16:00,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, ASSIGN because future has completed 2024-11-13T00:16:00,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:16:00,811 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,811 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:16:00,811 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,811 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:16:00,811 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,811 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,813 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,814 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:16:00,814 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041ae204889b5285cb06226a89ff8fa1 columnFamilyName A 2024-11-13T00:16:00,814 DEBUG [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:00,814 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(327): Store=041ae204889b5285cb06226a89ff8fa1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:16:00,814 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,815 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:16:00,815 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041ae204889b5285cb06226a89ff8fa1 columnFamilyName B 2024-11-13T00:16:00,815 DEBUG [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:00,816 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(327): Store=041ae204889b5285cb06226a89ff8fa1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:16:00,816 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,816 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:16:00,816 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041ae204889b5285cb06226a89ff8fa1 columnFamilyName C 2024-11-13T00:16:00,816 DEBUG [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:00,817 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(327): Store=041ae204889b5285cb06226a89ff8fa1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:16:00,817 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,817 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,818 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,818 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,819 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,819 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,820 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:16:00,821 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,822 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:16:00,823 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 041ae204889b5285cb06226a89ff8fa1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74519161, jitterRate=0.1104220300912857}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:16:00,823 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:00,823 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 041ae204889b5285cb06226a89ff8fa1: Running coprocessor pre-open hook at 1731456960812Writing region info on filesystem at 1731456960812Initializing all the Stores at 1731456960812Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456960812Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456960812Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456960812Cleaning up temporary data from old regions at 1731456960819 (+7 ms)Running coprocessor post-open hooks at 1731456960823 (+4 ms)Region opened successfully at 1731456960823 2024-11-13T00:16:00,824 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., pid=146, masterSystemTime=1731456960808 2024-11-13T00:16:00,825 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,825 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T00:16:00,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:16:00,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:16:00,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-13T00:16:00,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 in 170 msec 2024-11-13T00:16:00,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-13T00:16:00,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, ASSIGN in 326 msec 2024-11-13T00:16:00,831 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:16:00,831 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456960831"}]},"ts":"1731456960831"} 2024-11-13T00:16:00,832 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T00:16:00,832 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:16:00,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-13T00:16:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T00:16:01,836 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:01,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:01,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:01,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:01,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7ef3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:01,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eaae2f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:01,841 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:01,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:01,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:16:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,41869,1731456803739 2024-11-13T00:16:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d0b39e7 2024-11-13T00:16:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:16:01,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:16:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T00:16:01,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:01,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742416_1592 (size=1097) 2024-11-13T00:16:02,256 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T00:16:02,256 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T00:16:02,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:16:02,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, REOPEN/MOVE}] 2024-11-13T00:16:02,260 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, REOPEN/MOVE 2024-11-13T00:16:02,261 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, REOPEN/MOVE because future has completed 2024-11-13T00:16:02,262 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:16:02,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:16:02,414 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,414 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:16:02,414 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 041ae204889b5285cb06226a89ff8fa1, disabling compactions & flushes 2024-11-13T00:16:02,414 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,414 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,414 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. after waiting 0 ms 2024-11-13T00:16:02,414 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,417 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T00:16:02,418 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,418 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 041ae204889b5285cb06226a89ff8fa1: Waiting for close lock at 1731456962414Running coprocessor pre-close hooks at 1731456962414Disabling compacts and flushes for region at 1731456962414Disabling writes for close at 1731456962414Writing region close event to WAL at 1731456962415 (+1 ms)Running coprocessor post-close hooks at 1731456962418 (+3 ms)Closed at 1731456962418 2024-11-13T00:16:02,418 WARN [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 041ae204889b5285cb06226a89ff8fa1 to self. 2024-11-13T00:16:02,419 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=CLOSED 2024-11-13T00:16:02,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:16:02,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-13T00:16:02,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 in 159 msec 2024-11-13T00:16:02,423 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, REOPEN/MOVE; state=CLOSED, location=cb28580903fb,45273,1731456804594; forceNewPlan=false, retain=true 2024-11-13T00:16:02,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=OPENING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, REOPEN/MOVE because future has completed 2024-11-13T00:16:02,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:16:02,728 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,728 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:16:02,729 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,729 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:16:02,729 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,729 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,730 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,730 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:16:02,731 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041ae204889b5285cb06226a89ff8fa1 columnFamilyName A 2024-11-13T00:16:02,731 DEBUG [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:02,732 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(327): Store=041ae204889b5285cb06226a89ff8fa1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:16:02,732 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,732 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:16:02,733 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041ae204889b5285cb06226a89ff8fa1 columnFamilyName B 2024-11-13T00:16:02,733 DEBUG [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:02,733 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(327): Store=041ae204889b5285cb06226a89ff8fa1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:16:02,733 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,733 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T00:16:02,734 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 041ae204889b5285cb06226a89ff8fa1 columnFamilyName C 2024-11-13T00:16:02,734 DEBUG [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:02,734 INFO [StoreOpener-041ae204889b5285cb06226a89ff8fa1-1 {}] regionserver.HStore(327): Store=041ae204889b5285cb06226a89ff8fa1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:16:02,734 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,734 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,734 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,735 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,736 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,736 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,736 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:16:02,737 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,737 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 041ae204889b5285cb06226a89ff8fa1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64624697, jitterRate=-0.03701697289943695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:16:02,738 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,738 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 041ae204889b5285cb06226a89ff8fa1: Running coprocessor pre-open hook at 1731456962729Writing region info on filesystem at 1731456962729Initializing all the Stores at 1731456962729Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456962729Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456962730 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731456962730Cleaning up temporary data from old regions at 1731456962736 (+6 ms)Running coprocessor post-open hooks at 1731456962738 (+2 ms)Region opened successfully at 1731456962738 2024-11-13T00:16:02,739 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., pid=151, masterSystemTime=1731456962726 2024-11-13T00:16:02,740 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,740 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,741 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=OPEN, openSeqNum=5, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:16:02,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-13T00:16:02,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 in 168 msec 2024-11-13T00:16:02,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T00:16:02,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, REOPEN/MOVE in 485 msec 2024-11-13T00:16:02,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-13T00:16:02,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-13T00:16:02,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-13T00:16:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-13T00:16:02,749 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:02,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361af30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae4e117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,753 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7ce54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d69f2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,756 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475b9a01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ca4a58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e335b09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636adb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42595007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791ad110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66788c4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,764 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51abe8cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e6f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4c5e4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,768 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5f230e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9e337d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,771 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a991a2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485f8861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,41869,-1 for getting cluster id 2024-11-13T00:16:02,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:16:02,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2df54b55-b6b1-4607-8c19-554f8a3d512f' 2024-11-13T00:16:02,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2df54b55-b6b1-4607-8c19-554f8a3d512f" 2024-11-13T00:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a55ecf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,41869,-1] 2024-11-13T00:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:16:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:02,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:16:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500b1f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:16:02,775 DEBUG [Thread-2616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,775 DEBUG [Thread-2633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,775 DEBUG [Thread-2631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,775 DEBUG [Thread-2620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,776 DEBUG [Thread-2625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,776 DEBUG [Thread-2614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:02,776 DEBUG [Thread-2622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,777 DEBUG [Thread-2627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,777 DEBUG [Thread-2614 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,777 DEBUG [Thread-2614 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,777 DEBUG [Thread-2629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,778 DEBUG [Thread-2618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:16:02,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,45273,1731456804594, seqNum=-1] 2024-11-13T00:16:02,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:16:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T00:16:02,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:16:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:02,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:02,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5] 2024-11-13T00:16:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:02,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:16:02,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:02,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:02,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:02,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:02,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:02,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:02,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4cd83e4eac343078623b122b993fa1e_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456962786/Put/seqid=0 2024-11-13T00:16:02,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456962804, max timestamp=1731456962814Num uniques 859; 2024-11-13T00:16:02,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:02,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456962804, max timestamp=1731456962814Num uniques 859; 2024-11-13T00:16:02,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456962804, max timestamp=1731456962814Num uniques 859; 2024-11-13T00:16:02,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:02,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:02,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742417_1593 (size=17034) 2024-11-13T00:16:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457022832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457022837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457022837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457022838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457022838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:02,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:02,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:02,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457022946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457022946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457022946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457022946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457022946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457023156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457023156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457023156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457023157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457023157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,221 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:03,225 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4cd83e4eac343078623b122b993fa1e_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4cd83e4eac343078623b122b993fa1e_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:03,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/4f6a98d0a66c4f59b0976156ceeef3ec, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:03,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/4f6a98d0a66c4f59b0976156ceeef3ec is 175, key is test_row_0/A:col10/1731456962786/Put/seqid=0 2024-11-13T00:16:03,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742418_1594 (size=48139) 2024-11-13T00:16:03,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:03,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:03,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:03,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457023466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457023466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457023466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457023466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457023467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/4f6a98d0a66c4f59b0976156ceeef3ec 2024-11-13T00:16:03,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/aca9d3ff91754ad59920f1283c36d3b2 is 50, key is test_row_0/B:col10/1731456962786/Put/seqid=0 2024-11-13T00:16:03,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742419_1595 (size=12001) 2024-11-13T00:16:03,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/aca9d3ff91754ad59920f1283c36d3b2 2024-11-13T00:16:03,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/06637d47c4fb4032a7f7f83791b6b9f8 is 50, key is test_row_0/C:col10/1731456962786/Put/seqid=0 2024-11-13T00:16:03,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742420_1596 (size=12001) 2024-11-13T00:16:03,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/06637d47c4fb4032a7f7f83791b6b9f8 2024-11-13T00:16:03,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/4f6a98d0a66c4f59b0976156ceeef3ec as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec 2024-11-13T00:16:03,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:03,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec, entries=250, sequenceid=15, filesize=47.0 K 2024-11-13T00:16:03,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/aca9d3ff91754ad59920f1283c36d3b2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/aca9d3ff91754ad59920f1283c36d3b2 2024-11-13T00:16:03,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/aca9d3ff91754ad59920f1283c36d3b2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T00:16:03,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/06637d47c4fb4032a7f7f83791b6b9f8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/06637d47c4fb4032a7f7f83791b6b9f8 2024-11-13T00:16:03,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/06637d47c4fb4032a7f7f83791b6b9f8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T00:16:03,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 041ae204889b5285cb06226a89ff8fa1 in 926ms, sequenceid=15, compaction requested=false 2024-11-13T00:16:03,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:03,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T00:16:03,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:03,853 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T00:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:03,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137233d25f570343048fb69a6eef57481d_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456962836/Put/seqid=0 2024-11-13T00:16:03,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742421_1597 (size=12154) 2024-11-13T00:16:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:03,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:03,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457023985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457023986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457023986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457023987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:03,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457023989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457024096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457024096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457024096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:04,281 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137233d25f570343048fb69a6eef57481d_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137233d25f570343048fb69a6eef57481d_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:04,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/38f3237b883e4376a45c66c019187da4, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:04,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/38f3237b883e4376a45c66c019187da4 is 175, key is test_row_0/A:col10/1731456962836/Put/seqid=0 2024-11-13T00:16:04,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742422_1598 (size=30955) 2024-11-13T00:16:04,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457024305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457024306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457024306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,399 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:16:04,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457024616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457024616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457024616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:04,695 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/38f3237b883e4376a45c66c019187da4 2024-11-13T00:16:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/08fee221edb549b6a2378ed1db6d6e20 is 50, key is test_row_0/B:col10/1731456962836/Put/seqid=0 2024-11-13T00:16:04,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742423_1599 (size=12001) 2024-11-13T00:16:04,710 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/08fee221edb549b6a2378ed1db6d6e20 2024-11-13T00:16:04,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/a57297ed594e46da81a91d6735e50ed5 is 50, key is test_row_0/C:col10/1731456962836/Put/seqid=0 2024-11-13T00:16:04,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742424_1600 (size=12001) 2024-11-13T00:16:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457024996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:04,998 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457025005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:05,008 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:05,120 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/a57297ed594e46da81a91d6735e50ed5 2024-11-13T00:16:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/38f3237b883e4376a45c66c019187da4 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4 2024-11-13T00:16:05,127 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4, entries=150, sequenceid=52, filesize=30.2 K 2024-11-13T00:16:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/08fee221edb549b6a2378ed1db6d6e20 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/08fee221edb549b6a2378ed1db6d6e20 2024-11-13T00:16:05,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457025126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:05,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457025126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457025126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:05,131 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/08fee221edb549b6a2378ed1db6d6e20, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T00:16:05,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/a57297ed594e46da81a91d6735e50ed5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a57297ed594e46da81a91d6735e50ed5 2024-11-13T00:16:05,135 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a57297ed594e46da81a91d6735e50ed5, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T00:16:05,136 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 041ae204889b5285cb06226a89ff8fa1 in 1284ms, sequenceid=52, compaction requested=false 2024-11-13T00:16:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T00:16:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T00:16:05,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T00:16:05,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-11-13T00:16:05,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.3640 sec 2024-11-13T00:16:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:06,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T00:16:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:06,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139cd905f138254fcc9e3467943eb14bd3_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:06,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742425_1601 (size=14594) 2024-11-13T00:16:06,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457026174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457026175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457026176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457026286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457026286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457026286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457026496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457026496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457026496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:06,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139cd905f138254fcc9e3467943eb14bd3_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139cd905f138254fcc9e3467943eb14bd3_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:06,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/040b78b18e2648a480b621a3a3cfcf30, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:06,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/040b78b18e2648a480b621a3a3cfcf30 is 175, key is test_row_0/A:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:06,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742426_1602 (size=39549) 2024-11-13T00:16:06,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/040b78b18e2648a480b621a3a3cfcf30 2024-11-13T00:16:06,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/fb20f69a4a7a4e8494952de855815511 is 50, key is test_row_0/B:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:06,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742427_1603 (size=12001) 2024-11-13T00:16:06,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457026806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457026806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457026806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T00:16:06,946 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:06,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T00:16:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T00:16:06,949 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:06,949 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:06,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:06,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/fb20f69a4a7a4e8494952de855815511 2024-11-13T00:16:06,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/350f67f32fc546cbbcc5e1059eb415fe is 50, key is test_row_0/C:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:06,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742428_1604 (size=12001) 2024-11-13T00:16:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457027026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457027026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:07,028 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:07,029 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T00:16:07,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T00:16:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:07,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T00:16:07,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T00:16:07,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457027315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:07,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457027315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:07,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457027316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:07,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:07,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/350f67f32fc546cbbcc5e1059eb415fe 2024-11-13T00:16:07,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/040b78b18e2648a480b621a3a3cfcf30 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30 2024-11-13T00:16:07,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30, entries=200, sequenceid=64, filesize=38.6 K 2024-11-13T00:16:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/fb20f69a4a7a4e8494952de855815511 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/fb20f69a4a7a4e8494952de855815511 2024-11-13T00:16:07,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/fb20f69a4a7a4e8494952de855815511, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T00:16:07,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/350f67f32fc546cbbcc5e1059eb415fe as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/350f67f32fc546cbbcc5e1059eb415fe 2024-11-13T00:16:07,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/350f67f32fc546cbbcc5e1059eb415fe, entries=150, sequenceid=64, filesize=11.7 K 2024-11-13T00:16:07,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 041ae204889b5285cb06226a89ff8fa1 in 1259ms, sequenceid=64, compaction requested=true 2024-11-13T00:16:07,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:07,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:16:07,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:07,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:16:07,398 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:16:07,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:16:07,399 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:16:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:07,399 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:16:07,399 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:16:07,399 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/B is initiating minor compaction (all files) 2024-11-13T00:16:07,399 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/A is initiating minor compaction (all files) 2024-11-13T00:16:07,399 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/A in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,399 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/B in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,400 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=115.9 K 2024-11-13T00:16:07,400 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/aca9d3ff91754ad59920f1283c36d3b2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/08fee221edb549b6a2378ed1db6d6e20, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/fb20f69a4a7a4e8494952de855815511] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=35.2 K 2024-11-13T00:16:07,400 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,400 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30] 2024-11-13T00:16:07,400 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting aca9d3ff91754ad59920f1283c36d3b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731456962786 2024-11-13T00:16:07,400 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6a98d0a66c4f59b0976156ceeef3ec, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731456962782 2024-11-13T00:16:07,400 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f3237b883e4376a45c66c019187da4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731456962835 2024-11-13T00:16:07,400 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fee221edb549b6a2378ed1db6d6e20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731456962835 2024-11-13T00:16:07,401 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040b78b18e2648a480b621a3a3cfcf30, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731456963979 2024-11-13T00:16:07,401 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting fb20f69a4a7a4e8494952de855815511, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731456963984 2024-11-13T00:16:07,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T00:16:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,406 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T00:16:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:07,410 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:07,411 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:07,411 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ca41a50727e84f6eb0746cbcceadb8aa is 50, key is test_row_0/B:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134dcd98df38f2413dbf1dd102d721384a_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456966174/Put/seqid=0 2024-11-13T00:16:07,414 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bb5d1274a5d6472585e6365571d67792_041ae204889b5285cb06226a89ff8fa1 store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:07,416 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bb5d1274a5d6472585e6365571d67792_041ae204889b5285cb06226a89ff8fa1, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:07,416 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb5d1274a5d6472585e6365571d67792_041ae204889b5285cb06226a89ff8fa1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:07,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742429_1605 (size=12104) 2024-11-13T00:16:07,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742430_1606 (size=12154) 2024-11-13T00:16:07,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742431_1607 (size=4469) 2024-11-13T00:16:07,437 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#A#compaction#513 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:07,438 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/60e9256a49bd4b4a8aa468f19cf9d0ac is 175, key is test_row_0/A:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:07,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742432_1608 (size=31058) 2024-11-13T00:16:07,445 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/60e9256a49bd4b4a8aa468f19cf9d0ac as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/60e9256a49bd4b4a8aa468f19cf9d0ac 2024-11-13T00:16:07,449 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/A of 041ae204889b5285cb06226a89ff8fa1 into 60e9256a49bd4b4a8aa468f19cf9d0ac(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:07,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:07,449 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/A, priority=13, startTime=1731456967398; duration=0sec 2024-11-13T00:16:07,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:07,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:A 2024-11-13T00:16:07,449 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:16:07,450 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:16:07,450 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/C is initiating minor compaction (all files) 2024-11-13T00:16:07,450 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/C in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:07,450 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/06637d47c4fb4032a7f7f83791b6b9f8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a57297ed594e46da81a91d6735e50ed5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/350f67f32fc546cbbcc5e1059eb415fe] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=35.2 K 2024-11-13T00:16:07,450 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06637d47c4fb4032a7f7f83791b6b9f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731456962786 2024-11-13T00:16:07,450 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting a57297ed594e46da81a91d6735e50ed5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731456962835 2024-11-13T00:16:07,451 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350f67f32fc546cbbcc5e1059eb415fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731456963984 2024-11-13T00:16:07,456 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:07,456 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/aa25fcbeef044b9fbc9777a9249209c7 is 50, key is test_row_0/C:col10/1731456966137/Put/seqid=0 2024-11-13T00:16:07,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742433_1609 (size=12104) 2024-11-13T00:16:07,463 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/aa25fcbeef044b9fbc9777a9249209c7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/aa25fcbeef044b9fbc9777a9249209c7 2024-11-13T00:16:07,466 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/C of 041ae204889b5285cb06226a89ff8fa1 into aa25fcbeef044b9fbc9777a9249209c7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:07,467 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:07,467 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/C, priority=13, startTime=1731456967399; duration=0sec 2024-11-13T00:16:07,467 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:07,467 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:C 2024-11-13T00:16:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T00:16:07,830 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ca41a50727e84f6eb0746cbcceadb8aa as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca41a50727e84f6eb0746cbcceadb8aa 2024-11-13T00:16:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:07,834 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/B of 041ae204889b5285cb06226a89ff8fa1 into ca41a50727e84f6eb0746cbcceadb8aa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:07,834 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:07,834 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/B, priority=13, startTime=1731456967398; duration=0sec 2024-11-13T00:16:07,834 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:07,834 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:B 2024-11-13T00:16:07,835 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134dcd98df38f2413dbf1dd102d721384a_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134dcd98df38f2413dbf1dd102d721384a_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6b872f93bc914fbebc3f7cb2d1b758fc, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6b872f93bc914fbebc3f7cb2d1b758fc is 175, key is test_row_0/A:col10/1731456966174/Put/seqid=0 2024-11-13T00:16:07,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742434_1610 (size=30955) 2024-11-13T00:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T00:16:08,240 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6b872f93bc914fbebc3f7cb2d1b758fc 2024-11-13T00:16:08,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/78d1b9b02e9749038fffa563dfb01b0d is 50, key is test_row_0/B:col10/1731456966174/Put/seqid=0 2024-11-13T00:16:08,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742435_1611 (size=12001) 2024-11-13T00:16:08,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:08,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457028337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457028338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457028340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457028446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457028446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457028446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,656 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/78d1b9b02e9749038fffa563dfb01b0d 2024-11-13T00:16:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457028656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457028656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457028656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/73e83edc215f4ecf9a7a8a8fe3cce945 is 50, key is test_row_0/C:col10/1731456966174/Put/seqid=0 2024-11-13T00:16:08,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742436_1612 (size=12001) 2024-11-13T00:16:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457028966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457028966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457028966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,066 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/73e83edc215f4ecf9a7a8a8fe3cce945 2024-11-13T00:16:09,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6b872f93bc914fbebc3f7cb2d1b758fc as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc 2024-11-13T00:16:09,073 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc, entries=150, sequenceid=84, filesize=30.2 K 2024-11-13T00:16:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/78d1b9b02e9749038fffa563dfb01b0d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/78d1b9b02e9749038fffa563dfb01b0d 2024-11-13T00:16:09,078 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/78d1b9b02e9749038fffa563dfb01b0d, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T00:16:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/73e83edc215f4ecf9a7a8a8fe3cce945 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/73e83edc215f4ecf9a7a8a8fe3cce945 2024-11-13T00:16:09,081 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/73e83edc215f4ecf9a7a8a8fe3cce945, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T00:16:09,082 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 041ae204889b5285cb06226a89ff8fa1 in 1676ms, sequenceid=84, compaction requested=false 2024-11-13T00:16:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T00:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T00:16:09,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T00:16:09,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1340 sec 2024-11-13T00:16:09,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1390 sec 2024-11-13T00:16:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T00:16:09,096 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:09,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T00:16:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T00:16:09,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:09,099 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:09,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T00:16:09,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T00:16:09,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:09,252 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T00:16:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133bf07e0ee45c4451abf0b382c057296a_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456968328/Put/seqid=0 2024-11-13T00:16:09,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742437_1613 (size=12154) 2024-11-13T00:16:09,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:09,266 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133bf07e0ee45c4451abf0b382c057296a_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133bf07e0ee45c4451abf0b382c057296a_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:09,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6146a654d948467ba48da89bb6fff6b5, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:09,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6146a654d948467ba48da89bb6fff6b5 is 175, key is test_row_0/A:col10/1731456968328/Put/seqid=0 2024-11-13T00:16:09,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742438_1614 (size=30955) 2024-11-13T00:16:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T00:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:09,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:09,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456969492, max timestamp=1731456969504Num uniques 859; 2024-11-13T00:16:09,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456969492, max timestamp=1731456969504Num uniques 859; 2024-11-13T00:16:09,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:09,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:09,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456969492, max timestamp=1731456969504Num uniques 859; 2024-11-13T00:16:09,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:09,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457029562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457029562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457029562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,671 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6146a654d948467ba48da89bb6fff6b5 2024-11-13T00:16:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/c17c0ee0d9c745e1989e5a2b7bcfffb0 is 50, key is test_row_0/B:col10/1731456968328/Put/seqid=0 2024-11-13T00:16:09,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457029676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457029676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457029676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742439_1615 (size=12001) 2024-11-13T00:16:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T00:16:09,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457029886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457029886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457029886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,083 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/c17c0ee0d9c745e1989e5a2b7bcfffb0 2024-11-13T00:16:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7be1e168b4254743a86f73558b7de999 is 50, key is test_row_0/C:col10/1731456968328/Put/seqid=0 2024-11-13T00:16:10,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742440_1616 (size=12001) 2024-11-13T00:16:10,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457030196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457030196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457030196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T00:16:10,494 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7be1e168b4254743a86f73558b7de999 2024-11-13T00:16:10,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/6146a654d948467ba48da89bb6fff6b5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5 2024-11-13T00:16:10,500 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5, entries=150, sequenceid=99, filesize=30.2 K 2024-11-13T00:16:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/c17c0ee0d9c745e1989e5a2b7bcfffb0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/c17c0ee0d9c745e1989e5a2b7bcfffb0 2024-11-13T00:16:10,504 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/c17c0ee0d9c745e1989e5a2b7bcfffb0, entries=150, sequenceid=99, filesize=11.7 K 2024-11-13T00:16:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7be1e168b4254743a86f73558b7de999 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7be1e168b4254743a86f73558b7de999 2024-11-13T00:16:10,508 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7be1e168b4254743a86f73558b7de999, entries=150, sequenceid=99, filesize=11.7 K 2024-11-13T00:16:10,508 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 041ae204889b5285cb06226a89ff8fa1 in 1257ms, sequenceid=99, compaction requested=true 2024-11-13T00:16:10,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:10,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:10,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T00:16:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T00:16:10,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T00:16:10,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-13T00:16:10,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4150 sec 2024-11-13T00:16:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:10,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T00:16:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:10,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2414e1d36ce4e65a82b3eb7903c6a3c_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:10,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742441_1617 (size=17234) 2024-11-13T00:16:10,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457030722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457030728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457030728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457030836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457030836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:10,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457030836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457031046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457031046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457031046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457031056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457031056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,060 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:11,060 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,123 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:11,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2414e1d36ce4e65a82b3eb7903c6a3c_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2414e1d36ce4e65a82b3eb7903c6a3c_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:11,127 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/7dfe2eac949a4727b9aa66f4b7a5bc5e, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:11,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/7dfe2eac949a4727b9aa66f4b7a5bc5e is 175, key is test_row_0/A:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:11,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742442_1618 (size=48339) 2024-11-13T00:16:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T00:16:11,246 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:11,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T00:16:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T00:16:11,248 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:11,249 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:11,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T00:16:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457031355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457031356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457031356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/7dfe2eac949a4727b9aa66f4b7a5bc5e 2024-11-13T00:16:11,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/233b12fd490a4d0d8a154dd2a87e5ce9 is 50, key is test_row_0/B:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:11,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742443_1619 (size=12101) 2024-11-13T00:16:11,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T00:16:11,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:11,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:11,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457031866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457031866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457031866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T00:16:11,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/233b12fd490a4d0d8a154dd2a87e5ce9 2024-11-13T00:16:11,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/6dc779d3369e45408647f27272e15105 is 50, key is test_row_0/C:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:11,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742444_1620 (size=12101) 2024-11-13T00:16:12,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:12,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:12,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:12,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/6dc779d3369e45408647f27272e15105 2024-11-13T00:16:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/7dfe2eac949a4727b9aa66f4b7a5bc5e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e 2024-11-13T00:16:12,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e, entries=250, sequenceid=136, filesize=47.2 K 2024-11-13T00:16:12,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/233b12fd490a4d0d8a154dd2a87e5ce9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/233b12fd490a4d0d8a154dd2a87e5ce9 2024-11-13T00:16:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/233b12fd490a4d0d8a154dd2a87e5ce9, entries=150, sequenceid=136, filesize=11.8 K 2024-11-13T00:16:12,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/6dc779d3369e45408647f27272e15105 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6dc779d3369e45408647f27272e15105 2024-11-13T00:16:12,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6dc779d3369e45408647f27272e15105, entries=150, sequenceid=136, filesize=11.8 K 2024-11-13T00:16:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 041ae204889b5285cb06226a89ff8fa1 in 1666ms, sequenceid=136, compaction requested=true 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:12,377 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:16:12,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:12,377 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:16:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,378 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141307 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:16:12,378 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:16:12,378 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/A is initiating minor compaction (all files) 2024-11-13T00:16:12,378 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/B is initiating minor compaction (all files) 2024-11-13T00:16:12,378 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/A in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,378 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/B in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,378 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/60e9256a49bd4b4a8aa468f19cf9d0ac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=138.0 K 2024-11-13T00:16:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,378 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca41a50727e84f6eb0746cbcceadb8aa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/78d1b9b02e9749038fffa563dfb01b0d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/c17c0ee0d9c745e1989e5a2b7bcfffb0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/233b12fd490a4d0d8a154dd2a87e5ce9] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=47.1 K 2024-11-13T00:16:12,378 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,378 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/60e9256a49bd4b4a8aa468f19cf9d0ac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e] 2024-11-13T00:16:12,379 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ca41a50727e84f6eb0746cbcceadb8aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731456963984 2024-11-13T00:16:12,379 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60e9256a49bd4b4a8aa468f19cf9d0ac, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731456963984 2024-11-13T00:16:12,379 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b872f93bc914fbebc3f7cb2d1b758fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731456966168 2024-11-13T00:16:12,379 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d1b9b02e9749038fffa563dfb01b0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731456966168 2024-11-13T00:16:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,379 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6146a654d948467ba48da89bb6fff6b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731456968328 2024-11-13T00:16:12,379 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting c17c0ee0d9c745e1989e5a2b7bcfffb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731456968328 2024-11-13T00:16:12,380 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dfe2eac949a4727b9aa66f4b7a5bc5e, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731456969512 2024-11-13T00:16:12,384 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 233b12fd490a4d0d8a154dd2a87e5ce9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731456969512 2024-11-13T00:16:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T00:16:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,392 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,394 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#B#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,394 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411134217c893dd614cd9bb0bd01a456732e2_041ae204889b5285cb06226a89ff8fa1 store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:12,394 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ca21b5a9f29844d88e198a5ad872958f is 50, key is test_row_0/B:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,397 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411134217c893dd614cd9bb0bd01a456732e2_041ae204889b5285cb06226a89ff8fa1, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:12,397 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134217c893dd614cd9bb0bd01a456732e2_041ae204889b5285cb06226a89ff8fa1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742445_1621 (size=12341) 2024-11-13T00:16:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742446_1622 (size=4469) 2024-11-13T00:16:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T00:16:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,470 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T00:16:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139a44acf707374ad18a7b893b300f3ea4_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456970720/Put/seqid=0 2024-11-13T00:16:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742447_1623 (size=9814) 2024-11-13T00:16:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,806 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ca21b5a9f29844d88e198a5ad872958f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca21b5a9f29844d88e198a5ad872958f 2024-11-13T00:16:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,810 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/B of 041ae204889b5285cb06226a89ff8fa1 into ca21b5a9f29844d88e198a5ad872958f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:12,810 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,810 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/B, priority=12, startTime=1731456972377; duration=0sec 2024-11-13T00:16:12,810 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:12,810 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:B 2024-11-13T00:16:12,811 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,812 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T00:16:12,812 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#A#compaction#525 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:12,812 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/C is initiating minor compaction (all files) 2024-11-13T00:16:12,812 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/C in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:12,812 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/aa25fcbeef044b9fbc9777a9249209c7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/73e83edc215f4ecf9a7a8a8fe3cce945, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7be1e168b4254743a86f73558b7de999, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6dc779d3369e45408647f27272e15105] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=47.1 K 2024-11-13T00:16:12,813 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/8660b196f9134cc39877e7ac2feaad4c is 175, key is test_row_0/A:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,813 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting aa25fcbeef044b9fbc9777a9249209c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731456963984 2024-11-13T00:16:12,814 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e83edc215f4ecf9a7a8a8fe3cce945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731456966168 2024-11-13T00:16:12,814 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be1e168b4254743a86f73558b7de999, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731456968328 2024-11-13T00:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,815 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc779d3369e45408647f27272e15105, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731456969512 2024-11-13T00:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742448_1624 (size=31295) 2024-11-13T00:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,824 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/8660b196f9134cc39877e7ac2feaad4c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/8660b196f9134cc39877e7ac2feaad4c 2024-11-13T00:16:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,828 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:12,829 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/9d520e18bbf6473397db6c6ae55002e9 is 50, key is test_row_0/C:col10/1731456970711/Put/seqid=0 2024-11-13T00:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,831 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/A of 041ae204889b5285cb06226a89ff8fa1 into 8660b196f9134cc39877e7ac2feaad4c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:12,831 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,831 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/A, priority=12, startTime=1731456972377; duration=0sec 2024-11-13T00:16:12,831 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:12,831 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:A 2024-11-13T00:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742449_1625 (size=12341) 2024-11-13T00:16:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,839 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/9d520e18bbf6473397db6c6ae55002e9 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/9d520e18bbf6473397db6c6ae55002e9 2024-11-13T00:16:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,845 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/C of 041ae204889b5285cb06226a89ff8fa1 into 9d520e18bbf6473397db6c6ae55002e9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:12,845 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:12,845 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/C, priority=12, startTime=1731456972377; duration=0sec 2024-11-13T00:16:12,845 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:12,845 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:C 2024-11-13T00:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:12,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:12,887 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139a44acf707374ad18a7b893b300f3ea4_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139a44acf707374ad18a7b893b300f3ea4_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/243c1fda3812456a993cccce4d62ca25, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:12,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/243c1fda3812456a993cccce4d62ca25 is 175, key is test_row_0/A:col10/1731456970720/Put/seqid=0 2024-11-13T00:16:12,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742450_1626 (size=22461) 2024-11-13T00:16:12,894 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/243c1fda3812456a993cccce4d62ca25 2024-11-13T00:16:12,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/5f9722d0ca3a4714b20de41699e3c3ca is 50, key is test_row_0/B:col10/1731456970720/Put/seqid=0 2024-11-13T00:16:12,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742451_1627 (size=9757) 2024-11-13T00:16:12,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456972897, max timestamp=1731456972917Num uniques 859; 2024-11-13T00:16:12,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:12,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456972897, max timestamp=1731456972916Num uniques 859; 2024-11-13T00:16:12,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:12,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456972897, max timestamp=1731456972916Num uniques 859; 2024-11-13T00:16:12,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:12,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456972943, max timestamp=1731456972952Num uniques 1018; 2024-11-13T00:16:12,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T00:16:12,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456972943, max timestamp=1731456972952Num uniques 1018; 2024-11-13T00:16:12,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T00:16:12,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456972943, max timestamp=1731456972952Num uniques 1018; 2024-11-13T00:16:12,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T00:16:13,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457033013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457033013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457033014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457033126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457033126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457033126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,315 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/5f9722d0ca3a4714b20de41699e3c3ca 2024-11-13T00:16:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/10b87bf8d95e4a468f9174c4cf5cd450 is 50, key is test_row_0/C:col10/1731456970720/Put/seqid=0 2024-11-13T00:16:13,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742452_1628 (size=9757) 2024-11-13T00:16:13,327 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/10b87bf8d95e4a468f9174c4cf5cd450 2024-11-13T00:16:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/243c1fda3812456a993cccce4d62ca25 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25 2024-11-13T00:16:13,334 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25, entries=100, sequenceid=145, filesize=21.9 K 2024-11-13T00:16:13,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/5f9722d0ca3a4714b20de41699e3c3ca as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5f9722d0ca3a4714b20de41699e3c3ca 2024-11-13T00:16:13,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457033336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457033336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457033336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,339 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5f9722d0ca3a4714b20de41699e3c3ca, entries=100, sequenceid=145, filesize=9.5 K 2024-11-13T00:16:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/10b87bf8d95e4a468f9174c4cf5cd450 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/10b87bf8d95e4a468f9174c4cf5cd450 2024-11-13T00:16:13,343 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/10b87bf8d95e4a468f9174c4cf5cd450, entries=100, sequenceid=145, filesize=9.5 K 2024-11-13T00:16:13,343 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 041ae204889b5285cb06226a89ff8fa1 in 873ms, sequenceid=145, compaction requested=false 2024-11-13T00:16:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T00:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T00:16:13,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T00:16:13,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-11-13T00:16:13,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1020 sec 2024-11-13T00:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T00:16:13,396 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:13,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T00:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T00:16:13,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:13,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:13,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T00:16:13,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:13,552 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:13,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135cfa07d27c2d446a89464a7ca8b9dd69_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456972993/Put/seqid=0 2024-11-13T00:16:13,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742453_1629 (size=12304) 2024-11-13T00:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:13,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457033657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457033657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457033658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T00:16:13,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457033766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457033766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457033766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:13,966 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135cfa07d27c2d446a89464a7ca8b9dd69_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135cfa07d27c2d446a89464a7ca8b9dd69_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:13,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/47fdb66e14554c60810a55a103d6b3a8, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:13,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/47fdb66e14554c60810a55a103d6b3a8 is 175, key is test_row_0/A:col10/1731456972993/Put/seqid=0 2024-11-13T00:16:13,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742454_1630 (size=31105) 2024-11-13T00:16:13,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457033976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457033976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457033976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T00:16:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457034286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457034286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457034286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,371 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/47fdb66e14554c60810a55a103d6b3a8 2024-11-13T00:16:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/9e94f48152024c4396e856421640acd8 is 50, key is test_row_0/B:col10/1731456972993/Put/seqid=0 2024-11-13T00:16:14,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742455_1631 (size=12151) 2024-11-13T00:16:14,382 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/9e94f48152024c4396e856421640acd8 2024-11-13T00:16:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/6666186f4611414d94dfec84033ef0ee is 50, key is test_row_0/C:col10/1731456972993/Put/seqid=0 2024-11-13T00:16:14,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742456_1632 (size=12151) 2024-11-13T00:16:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T00:16:14,794 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/6666186f4611414d94dfec84033ef0ee 2024-11-13T00:16:14,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457034796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:14,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/47fdb66e14554c60810a55a103d6b3a8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8 2024-11-13T00:16:14,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457034796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,804 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8, entries=150, sequenceid=202, filesize=30.4 K 2024-11-13T00:16:14,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/9e94f48152024c4396e856421640acd8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9e94f48152024c4396e856421640acd8 2024-11-13T00:16:14,808 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9e94f48152024c4396e856421640acd8, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T00:16:14,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457034806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/6666186f4611414d94dfec84033ef0ee as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6666186f4611414d94dfec84033ef0ee 2024-11-13T00:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:14,813 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6666186f4611414d94dfec84033ef0ee, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T00:16:14,813 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for 041ae204889b5285cb06226a89ff8fa1 in 1261ms, sequenceid=202, compaction requested=true 2024-11-13T00:16:14,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:14,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:14,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T00:16:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T00:16:14,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T00:16:14,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-13T00:16:14,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4200 sec 2024-11-13T00:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T00:16:15,546 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:15,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-13T00:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T00:16:15,549 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:15,549 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:15,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T00:16:15,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T00:16:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:15,703 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T00:16:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:15,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:15,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366a6d8d4fa7c4d969dd5aecd0f845474_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456973656/Put/seqid=0 2024-11-13T00:16:15,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742457_1633 (size=12304) 2024-11-13T00:16:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:15,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:15,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456975833, max timestamp=1731456975844Num uniques 859; 2024-11-13T00:16:15,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:15,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456975833, max timestamp=1731456975844Num uniques 859; 2024-11-13T00:16:15,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456975833, max timestamp=1731456975844Num uniques 859; 2024-11-13T00:16:15,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:15,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T00:16:15,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457035876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457035882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:15,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457035883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457035986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:15,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457035995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:15,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457035996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:16,120 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366a6d8d4fa7c4d969dd5aecd0f845474_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366a6d8d4fa7c4d969dd5aecd0f845474_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/50292e637dc64f9693aa72dadfb4f2cf, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:16,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/50292e637dc64f9693aa72dadfb4f2cf is 175, key is test_row_0/A:col10/1731456973656/Put/seqid=0 2024-11-13T00:16:16,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742458_1634 (size=31105) 2024-11-13T00:16:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T00:16:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457036196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457036206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457036206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457036506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457036516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457036516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:16,543 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/50292e637dc64f9693aa72dadfb4f2cf 2024-11-13T00:16:16,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/2d5b3cc973b343f5bf65e9c9c1c88200 is 50, key is test_row_0/B:col10/1731456973656/Put/seqid=0 2024-11-13T00:16:16,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742459_1635 (size=12151) 2024-11-13T00:16:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T00:16:16,954 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/2d5b3cc973b343f5bf65e9c9c1c88200 2024-11-13T00:16:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7725e1e13e6141a7a0a3d08e134eec9d is 50, key is test_row_0/C:col10/1731456973656/Put/seqid=0 2024-11-13T00:16:16,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742460_1636 (size=12151) 2024-11-13T00:16:17,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457037016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457037026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457037026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:17,364 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7725e1e13e6141a7a0a3d08e134eec9d 2024-11-13T00:16:17,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/50292e637dc64f9693aa72dadfb4f2cf as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf 2024-11-13T00:16:17,372 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf, entries=150, sequenceid=211, filesize=30.4 K 2024-11-13T00:16:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/2d5b3cc973b343f5bf65e9c9c1c88200 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/2d5b3cc973b343f5bf65e9c9c1c88200 2024-11-13T00:16:17,376 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/2d5b3cc973b343f5bf65e9c9c1c88200, entries=150, sequenceid=211, filesize=11.9 K 2024-11-13T00:16:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7725e1e13e6141a7a0a3d08e134eec9d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7725e1e13e6141a7a0a3d08e134eec9d 2024-11-13T00:16:17,380 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7725e1e13e6141a7a0a3d08e134eec9d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-13T00:16:17,381 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 041ae204889b5285cb06226a89ff8fa1 in 1679ms, sequenceid=211, compaction requested=true 2024-11-13T00:16:17,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:17,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:17,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-13T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-13T00:16:17,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T00:16:17,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-13T00:16:17,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8380 sec 2024-11-13T00:16:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T00:16:17,696 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:17,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-13T00:16:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T00:16:17,698 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:17,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:17,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T00:16:17,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T00:16:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:17,851 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-13T00:16:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T00:16:17,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111383239b5ee13c4cb5b1e67c33c1c4eabb_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456975868/Put/seqid=0 2024-11-13T00:16:17,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742461_1637 (size=12304) 2024-11-13T00:16:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:17,869 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111383239b5ee13c4cb5b1e67c33c1c4eabb_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383239b5ee13c4cb5b1e67c33c1c4eabb_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d7de3157c0d64b6792fc3d64cbff7787, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d7de3157c0d64b6792fc3d64cbff7787 is 175, key is test_row_0/A:col10/1731456975868/Put/seqid=0 2024-11-13T00:16:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742462_1638 (size=31105) 2024-11-13T00:16:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T00:16:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457038048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457038049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457038050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457038166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457038166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457038166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,287 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d7de3157c0d64b6792fc3d64cbff7787 2024-11-13T00:16:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/52cf3481a24040f2ad72d0b1044dd8f1 is 50, key is test_row_0/B:col10/1731456975868/Put/seqid=0 2024-11-13T00:16:18,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742463_1639 (size=12151) 2024-11-13T00:16:18,299 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/52cf3481a24040f2ad72d0b1044dd8f1 2024-11-13T00:16:18,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/65362726e5764130abf312a4910e5f4b is 50, key is test_row_0/C:col10/1731456975868/Put/seqid=0 2024-11-13T00:16:18,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742464_1640 (size=12151) 2024-11-13T00:16:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T00:16:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457038376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457038376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457038376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457038685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457038686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457038686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:18,713 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/65362726e5764130abf312a4910e5f4b 2024-11-13T00:16:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d7de3157c0d64b6792fc3d64cbff7787 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787 2024-11-13T00:16:18,720 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787, entries=150, sequenceid=248, filesize=30.4 K 2024-11-13T00:16:18,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/52cf3481a24040f2ad72d0b1044dd8f1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/52cf3481a24040f2ad72d0b1044dd8f1 2024-11-13T00:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,724 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/52cf3481a24040f2ad72d0b1044dd8f1, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T00:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/65362726e5764130abf312a4910e5f4b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/65362726e5764130abf312a4910e5f4b 2024-11-13T00:16:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,728 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/65362726e5764130abf312a4910e5f4b, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T00:16:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,729 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 041ae204889b5285cb06226a89ff8fa1 in 878ms, sequenceid=248, compaction requested=true 2024-11-13T00:16:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-13T00:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-13T00:16:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T00:16:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-11-13T00:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0370 sec 2024-11-13T00:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T00:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,836 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:18,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-13T00:16:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T00:16:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:18,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T00:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-13T00:16:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,994 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T00:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f2951df6753494f86a948b1db827a0f_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_1/A:col10/1731456978041/Put/seqid=0 2024-11-13T00:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742465_1641 (size=9814) 2024-11-13T00:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T00:16:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456979229, max timestamp=1731456979238Num uniques 859; 2024-11-13T00:16:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731456979229, max timestamp=1731456979238Num uniques 859; 2024-11-13T00:16:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T00:16:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456979229, max timestamp=1731456979238Num uniques 859; 2024-11-13T00:16:19,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457039276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456979261, max timestamp=1731456979281Num uniques 1068; 2024-11-13T00:16:19,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T00:16:19,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456979261, max timestamp=1731456979281Num uniques 1018; 2024-11-13T00:16:19,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T00:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731456979261, max timestamp=1731456979281Num uniques 1018; 2024-11-13T00:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-13T00:16:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457039328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457039332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457039386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,410 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f2951df6753494f86a948b1db827a0f_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f2951df6753494f86a948b1db827a0f_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:19,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d18243476bf74d69a20848baa1005a0b, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d18243476bf74d69a20848baa1005a0b is 175, key is test_row_1/A:col10/1731456978041/Put/seqid=0 2024-11-13T00:16:19,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742466_1642 (size=22461) 2024-11-13T00:16:19,415 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d18243476bf74d69a20848baa1005a0b 2024-11-13T00:16:19,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/5559ad5d04ed490f8f91a183e2dc9227 is 50, key is test_row_1/B:col10/1731456978041/Put/seqid=0 2024-11-13T00:16:19,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742467_1643 (size=9757) 2024-11-13T00:16:19,424 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/5559ad5d04ed490f8f91a183e2dc9227 2024-11-13T00:16:19,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7ba92dd7c39e4be3a11ab2f91b4c8967 is 50, key is test_row_1/C:col10/1731456978041/Put/seqid=0 2024-11-13T00:16:19,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742468_1644 (size=9757) 2024-11-13T00:16:19,435 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7ba92dd7c39e4be3a11ab2f91b4c8967 2024-11-13T00:16:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/d18243476bf74d69a20848baa1005a0b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b 2024-11-13T00:16:19,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457039436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,441 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b, entries=100, sequenceid=257, filesize=21.9 K 2024-11-13T00:16:19,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/5559ad5d04ed490f8f91a183e2dc9227 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5559ad5d04ed490f8f91a183e2dc9227 2024-11-13T00:16:19,445 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5559ad5d04ed490f8f91a183e2dc9227, entries=100, sequenceid=257, filesize=9.5 K 2024-11-13T00:16:19,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/7ba92dd7c39e4be3a11ab2f91b4c8967 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7ba92dd7c39e4be3a11ab2f91b4c8967 2024-11-13T00:16:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7ba92dd7c39e4be3a11ab2f91b4c8967, entries=100, sequenceid=257, filesize=9.5 K 2024-11-13T00:16:19,450 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 041ae204889b5285cb06226a89ff8fa1 in 455ms, sequenceid=257, compaction requested=true 2024-11-13T00:16:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-13T00:16:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-13T00:16:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:19,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-13T00:16:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:16:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:16:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:16:19,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-13T00:16:19,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 610 msec 2024-11-13T00:16:19,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 620 msec 2024-11-13T00:16:19,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371d27564393a4f01b0d835aead1ce3eb_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:19,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742469_1645 (size=14994) 2024-11-13T00:16:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T00:16:19,469 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:19,469 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T00:16:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-13T00:16:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:19,471 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T00:16:19,471 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T00:16:19,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T00:16:19,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457039485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457039596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457039596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:19,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:19,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457039645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:19,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457039806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:19,870 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371d27564393a4f01b0d835aead1ce3eb_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371d27564393a4f01b0d835aead1ce3eb_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:19,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/440ce5d2ecd24868ba76264d51140202, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:19,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/440ce5d2ecd24868ba76264d51140202 is 175, key is test_row_0/A:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:19,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742470_1646 (size=39949) 2024-11-13T00:16:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457039906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:19,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:19,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:19,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457039956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:20,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457040116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:20,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/440ce5d2ecd24868ba76264d51140202 2024-11-13T00:16:20,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/9ec8316b3ad5434d9208bb30ab7d83be is 50, key is test_row_0/B:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:20,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742471_1647 (size=12301) 2024-11-13T00:16:20,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457040416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:20,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457040465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:20,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:20,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457040626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:20,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/9ec8316b3ad5434d9208bb30ab7d83be 2024-11-13T00:16:20,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/34cfe164914c4fe7b6e00b2738fa2fbb is 50, key is test_row_0/C:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:20,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:16:20,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742472_1648 (size=12301) 2024-11-13T00:16:20,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/34cfe164914c4fe7b6e00b2738fa2fbb 2024-11-13T00:16:20,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/440ce5d2ecd24868ba76264d51140202 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202 2024-11-13T00:16:20,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202, entries=200, sequenceid=312, filesize=39.0 K 2024-11-13T00:16:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/9ec8316b3ad5434d9208bb30ab7d83be as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9ec8316b3ad5434d9208bb30ab7d83be 2024-11-13T00:16:20,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9ec8316b3ad5434d9208bb30ab7d83be, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T00:16:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/34cfe164914c4fe7b6e00b2738fa2fbb as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/34cfe164914c4fe7b6e00b2738fa2fbb 2024-11-13T00:16:20,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/34cfe164914c4fe7b6e00b2738fa2fbb, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T00:16:20,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 041ae204889b5285cb06226a89ff8fa1 in 1264ms, sequenceid=312, compaction requested=true 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:16:20,716 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:20,716 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:16:20,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,718 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 80609 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-13T00:16:20,718 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/B is initiating minor compaction (all files) 2024-11-13T00:16:20,718 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 209481 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-13T00:16:20,718 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/B in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,718 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/A is initiating minor compaction (all files) 2024-11-13T00:16:20,718 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/A in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,718 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca21b5a9f29844d88e198a5ad872958f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5f9722d0ca3a4714b20de41699e3c3ca, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9e94f48152024c4396e856421640acd8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/2d5b3cc973b343f5bf65e9c9c1c88200, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/52cf3481a24040f2ad72d0b1044dd8f1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5559ad5d04ed490f8f91a183e2dc9227, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9ec8316b3ad5434d9208bb30ab7d83be] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=78.7 K 2024-11-13T00:16:20,718 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/8660b196f9134cc39877e7ac2feaad4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=204.6 K 2024-11-13T00:16:20,718 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,718 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/8660b196f9134cc39877e7ac2feaad4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202] 2024-11-13T00:16:20,719 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ca21b5a9f29844d88e198a5ad872958f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731456969512 2024-11-13T00:16:20,719 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8660b196f9134cc39877e7ac2feaad4c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731456969512 2024-11-13T00:16:20,719 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243c1fda3812456a993cccce4d62ca25, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731456970720 2024-11-13T00:16:20,719 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f9722d0ca3a4714b20de41699e3c3ca, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731456970720 2024-11-13T00:16:20,720 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47fdb66e14554c60810a55a103d6b3a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731456972981 2024-11-13T00:16:20,720 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e94f48152024c4396e856421640acd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731456972981 2024-11-13T00:16:20,720 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50292e637dc64f9693aa72dadfb4f2cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731456973655 2024-11-13T00:16:20,720 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d5b3cc973b343f5bf65e9c9c1c88200, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731456973655 2024-11-13T00:16:20,721 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 52cf3481a24040f2ad72d0b1044dd8f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731456975868 2024-11-13T00:16:20,721 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7de3157c0d64b6792fc3d64cbff7787, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731456975868 2024-11-13T00:16:20,721 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 5559ad5d04ed490f8f91a183e2dc9227, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731456978041 2024-11-13T00:16:20,721 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18243476bf74d69a20848baa1005a0b, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731456978041 2024-11-13T00:16:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,721 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440ce5d2ecd24868ba76264d51140202, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731456979323 2024-11-13T00:16:20,721 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec8316b3ad5434d9208bb30ab7d83be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731456979323 2024-11-13T00:16:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,736 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#B#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:20,737 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/33d0b73ac0734c408a1706b08c9362b0 is 50, key is test_row_0/B:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,738 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,741 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111391c43ccfe026426b9c742d8c206f7445_041ae204889b5285cb06226a89ff8fa1 store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:20,744 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111391c43ccfe026426b9c742d8c206f7445_041ae204889b5285cb06226a89ff8fa1, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:20,744 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391c43ccfe026426b9c742d8c206f7445_041ae204889b5285cb06226a89ff8fa1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:20,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742473_1649 (size=12779) 2024-11-13T00:16:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,750 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/33d0b73ac0734c408a1706b08c9362b0 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/33d0b73ac0734c408a1706b08c9362b0 2024-11-13T00:16:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742474_1650 (size=4469) 2024-11-13T00:16:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,752 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#A#compaction#546 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,752 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/5b66d296246944c7b1aff3dc6899811c is 175, key is test_row_0/A:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,754 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/B of 041ae204889b5285cb06226a89ff8fa1 into 33d0b73ac0734c408a1706b08c9362b0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:20,754 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:20,754 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/B, priority=9, startTime=1731456980716; duration=0sec 2024-11-13T00:16:20,754 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:20,754 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:B 2024-11-13T00:16:20,755 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-13T00:16:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742475_1651 (size=31733) 2024-11-13T00:16:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,757 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 80609 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-13T00:16:20,757 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/C is initiating minor compaction (all files) 2024-11-13T00:16:20,757 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/C in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,758 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/9d520e18bbf6473397db6c6ae55002e9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/10b87bf8d95e4a468f9174c4cf5cd450, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6666186f4611414d94dfec84033ef0ee, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7725e1e13e6141a7a0a3d08e134eec9d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/65362726e5764130abf312a4910e5f4b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7ba92dd7c39e4be3a11ab2f91b4c8967, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/34cfe164914c4fe7b6e00b2738fa2fbb] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=78.7 K 2024-11-13T00:16:20,758 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d520e18bbf6473397db6c6ae55002e9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731456969512 2024-11-13T00:16:20,758 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b87bf8d95e4a468f9174c4cf5cd450, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731456970720 2024-11-13T00:16:20,760 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 6666186f4611414d94dfec84033ef0ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731456972981 2024-11-13T00:16:20,761 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7725e1e13e6141a7a0a3d08e134eec9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731456973655 2024-11-13T00:16:20,761 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 65362726e5764130abf312a4910e5f4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731456975868 2024-11-13T00:16:20,761 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba92dd7c39e4be3a11ab2f91b4c8967, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731456978041 2024-11-13T00:16:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,763 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 34cfe164914c4fe7b6e00b2738fa2fbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731456979323 2024-11-13T00:16:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,765 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/5b66d296246944c7b1aff3dc6899811c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/5b66d296246944c7b1aff3dc6899811c 2024-11-13T00:16:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,770 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/A of 041ae204889b5285cb06226a89ff8fa1 into 5b66d296246944c7b1aff3dc6899811c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:20,770 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,770 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/A, priority=9, startTime=1731456980716; duration=0sec 2024-11-13T00:16:20,770 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:20,771 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:A 2024-11-13T00:16:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,777 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#C#compaction#547 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,777 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/21a47cc410d34956999e3a72524af640 is 50, key is test_row_0/C:col10/1731456979323/Put/seqid=0 2024-11-13T00:16:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742476_1652 (size=12779) 2024-11-13T00:16:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45273 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T00:16:20,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:20,846 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T00:16:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136047afdca72f41828fe01674a4f850f2_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456979483/Put/seqid=0 2024-11-13T00:16:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742477_1653 (size=9914) 2024-11-13T00:16:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:21,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. as already flushing 2024-11-13T00:16:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456981138, max timestamp=1731456981152Num uniques 859; 2024-11-13T00:16:21,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:21,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456981138, max timestamp=1731456981152Num uniques 859; 2024-11-13T00:16:21,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:21,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731456981138, max timestamp=1731456981152Num uniques 859; 2024-11-13T00:16:21,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T00:16:21,186 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/21a47cc410d34956999e3a72524af640 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/21a47cc410d34956999e3a72524af640 2024-11-13T00:16:21,189 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/C of 041ae204889b5285cb06226a89ff8fa1 into 21a47cc410d34956999e3a72524af640(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:21,189 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:21,189 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/C, priority=9, startTime=1731456980716; duration=0sec 2024-11-13T00:16:21,190 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:21,190 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:C 2024-11-13T00:16:21,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456981189, max timestamp=1731456981204Num uniques 1018; 2024-11-13T00:16:21,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456981189, max timestamp=1731456981204Num uniques 1018; 2024-11-13T00:16:21,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:16:21,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731456981189, max timestamp=1731456981204Num uniques 1018; 2024-11-13T00:16:21,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:16:21,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T00:16:21,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731456981236, max timestamp=1731456981248Num uniques 1027; 2024-11-13T00:16:21,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T00:16:21,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731456981236, max timestamp=1731456981248Num uniques 1027; 2024-11-13T00:16:21,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T00:16:21,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731456981236, max timestamp=1731456981248Num uniques 1027; 2024-11-13T00:16:21,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-13T00:16:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,261 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136047afdca72f41828fe01674a4f850f2_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136047afdca72f41828fe01674a4f850f2_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:21,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3f915249ce1f4696af3b9c0617460af2, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3f915249ce1f4696af3b9c0617460af2 is 175, key is test_row_0/A:col10/1731456979483/Put/seqid=0 2024-11-13T00:16:21,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742478_1654 (size=22561) 2024-11-13T00:16:21,266 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3f915249ce1f4696af3b9c0617460af2 2024-11-13T00:16:21,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ef7f3a8990664fc39f79aa8c4d8da50e is 50, key is test_row_0/B:col10/1731456979483/Put/seqid=0 2024-11-13T00:16:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742479_1655 (size=9857) 2024-11-13T00:16:21,275 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ef7f3a8990664fc39f79aa8c4d8da50e 2024-11-13T00:16:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/01fbfb76c9fa400ba833ab659e199fcc is 50, key is test_row_0/C:col10/1731456979483/Put/seqid=0 2024-11-13T00:16:21,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742480_1656 (size=9857) 2024-11-13T00:16:21,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731456981281, max timestamp=1731456981294Num uniques 1036; 2024-11-13T00:16:21,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T00:16:21,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731456981281, max timestamp=1731456981294Num uniques 1036; 2024-11-13T00:16:21,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731456981281, max timestamp=1731456981294Num uniques 1036; 2024-11-13T00:16:21,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T00:16:21,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-13T00:16:21,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731456981317, max timestamp=1731456981338Num uniques 1095; 2024-11-13T00:16:21,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731456981317, max timestamp=1731456981338Num uniques 1095; 2024-11-13T00:16:21,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-13T00:16:21,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731456981317, max timestamp=1731456981338Num uniques 1095; 2024-11-13T00:16:21,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-13T00:16:21,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/cb28580903fb:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-13T00:16:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457041384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457041388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39212 deadline: 1731457041426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,429 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_1' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39198 deadline: 1731457041475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,477 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_2' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457041496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457041496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39186 deadline: 1731457041636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,639 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cb28580903fb,45273,1731456804594 for 'test_row_0' in 041ae204889b5285cb06226a89ff8fa1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T00:16:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,684 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/01fbfb76c9fa400ba833ab659e199fcc 2024-11-13T00:16:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3f915249ce1f4696af3b9c0617460af2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2 2024-11-13T00:16:21,691 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2, entries=100, sequenceid=322, filesize=22.0 K 2024-11-13T00:16:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/ef7f3a8990664fc39f79aa8c4d8da50e as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ef7f3a8990664fc39f79aa8c4d8da50e 2024-11-13T00:16:21,694 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ef7f3a8990664fc39f79aa8c4d8da50e, entries=100, sequenceid=322, filesize=9.6 K 2024-11-13T00:16:21,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/01fbfb76c9fa400ba833ab659e199fcc as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01fbfb76c9fa400ba833ab659e199fcc 2024-11-13T00:16:21,698 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01fbfb76c9fa400ba833ab659e199fcc, entries=100, sequenceid=322, filesize=9.6 K 2024-11-13T00:16:21,698 INFO [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 041ae204889b5285cb06226a89ff8fa1 in 852ms, sequenceid=322, compaction requested=false 2024-11-13T00:16:21,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:21,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:21,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-13T00:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-13T00:16:21,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-13T00:16:21,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2290 sec 2024-11-13T00:16:21,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.2330 sec 2024-11-13T00:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:21,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-13T00:16:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:16:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:16:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T00:16:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111330223f9620ca4ec8aac5be55faea2cb0_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742481_1657 (size=14994) 2024-11-13T00:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457041716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457041717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111330223f9620ca4ec8aac5be55faea2cb0_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111330223f9620ca4ec8aac5be55faea2cb0_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:21,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/649fd7f079514182ad8c36ec60b86d54, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:21,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/649fd7f079514182ad8c36ec60b86d54 is 175, key is test_row_0/A:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:21,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742482_1658 (size=39949) 2024-11-13T00:16:21,729 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/649fd7f079514182ad8c36ec60b86d54 2024-11-13T00:16:21,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/079c93affb7f484cb519a82bd449bcb8 is 50, key is test_row_0/B:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742483_1659 (size=12301) 2024-11-13T00:16:21,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457041826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457041826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457042036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457042036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:22,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/079c93affb7f484cb519a82bd449bcb8 2024-11-13T00:16:22,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/01e763f39e93493884e6a747bec15ae7 is 50, key is test_row_0/C:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:22,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742484_1660 (size=12301) 2024-11-13T00:16:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39188 deadline: 1731457042346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:22,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:16:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45273 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39258 deadline: 1731457042346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 2024-11-13T00:16:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 , the old value is region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T00:16:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., hostname=cb28580903fb,45273,1731456804594, seqNum=5 because the exception is null or not the one we care about 2024-11-13T00:16:22,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/01e763f39e93493884e6a747bec15ae7 2024-11-13T00:16:22,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/649fd7f079514182ad8c36ec60b86d54 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54 2024-11-13T00:16:22,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54, entries=200, sequenceid=430, filesize=39.0 K 2024-11-13T00:16:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/079c93affb7f484cb519a82bd449bcb8 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/079c93affb7f484cb519a82bd449bcb8 2024-11-13T00:16:22,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/079c93affb7f484cb519a82bd449bcb8, entries=150, sequenceid=430, filesize=12.0 K 2024-11-13T00:16:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/01e763f39e93493884e6a747bec15ae7 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01e763f39e93493884e6a747bec15ae7 2024-11-13T00:16:22,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01e763f39e93493884e6a747bec15ae7, entries=150, sequenceid=430, filesize=12.0 K 2024-11-13T00:16:22,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~437.34 KB/447840, currentSize=26.84 KB/27480 for 041ae204889b5285cb06226a89ff8fa1 in 857ms, sequenceid=430, compaction requested=true 2024-11-13T00:16:22,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T00:16:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T00:16:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:22,566 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:16:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 041ae204889b5285cb06226a89ff8fa1:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T00:16:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:22,566 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:16:22,567 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:16:22,567 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/A is initiating minor compaction (all files) 2024-11-13T00:16:22,567 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/A in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:22,567 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/5b66d296246944c7b1aff3dc6899811c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=92.0 K 2024-11-13T00:16:22,567 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:22,567 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. files: [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/5b66d296246944c7b1aff3dc6899811c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54] 2024-11-13T00:16:22,567 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:16:22,567 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/B is initiating minor compaction (all files) 2024-11-13T00:16:22,567 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/B in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:22,567 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/33d0b73ac0734c408a1706b08c9362b0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ef7f3a8990664fc39f79aa8c4d8da50e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/079c93affb7f484cb519a82bd449bcb8] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=34.1 K 2024-11-13T00:16:22,568 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b66d296246944c7b1aff3dc6899811c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731456979323 2024-11-13T00:16:22,568 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d0b73ac0734c408a1706b08c9362b0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731456979323 2024-11-13T00:16:22,568 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f915249ce1f4696af3b9c0617460af2, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731456979470 2024-11-13T00:16:22,568 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7f3a8990664fc39f79aa8c4d8da50e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731456979470 2024-11-13T00:16:22,568 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 079c93affb7f484cb519a82bd449bcb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731456981380 2024-11-13T00:16:22,568 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] compactions.Compactor(225): Compacting 649fd7f079514182ad8c36ec60b86d54, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731456981372 2024-11-13T00:16:22,575 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:22,577 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#B#compaction#555 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:22,577 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138d790951bafa4af5a666058920b39f2b_041ae204889b5285cb06226a89ff8fa1 store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:22,577 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/191947360e974a96a878eef2ab4612c6 is 50, key is test_row_0/B:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:22,580 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138d790951bafa4af5a666058920b39f2b_041ae204889b5285cb06226a89ff8fa1, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:22,580 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138d790951bafa4af5a666058920b39f2b_041ae204889b5285cb06226a89ff8fa1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:22,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742485_1661 (size=12881) 2024-11-13T00:16:22,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742486_1662 (size=4469) 2024-11-13T00:16:22,589 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#A#compaction#554 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:22,590 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/bf4a5af1df194c4f98cf3c0098856af1 is 175, key is test_row_0/A:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:22,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742487_1663 (size=31835) 2024-11-13T00:16:22,778 INFO [Thread-2629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2629. 2024-11-13T00:16:22,778 DEBUG [Thread-2629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,778 DEBUG [Thread-2629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,778 DEBUG [Thread-2629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,779 INFO [Thread-2631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2631. 2024-11-13T00:16:22,779 DEBUG [Thread-2631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,779 DEBUG [Thread-2631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,779 DEBUG [Thread-2631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,779 INFO [Thread-2633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2633. 2024-11-13T00:16:22,779 DEBUG [Thread-2633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,779 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,780 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,780 INFO [Thread-2625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2625. 2024-11-13T00:16:22,780 DEBUG [Thread-2625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,780 DEBUG [Thread-2625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,780 DEBUG [Thread-2625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,781 INFO [Thread-2627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2627. 2024-11-13T00:16:22,781 DEBUG [Thread-2627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,781 DEBUG [Thread-2627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,781 DEBUG [Thread-2627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:16:22,858 INFO [Thread-2618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2618. 2024-11-13T00:16:22,858 DEBUG [Thread-2618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,858 DEBUG [Thread-2618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,858 INFO [Thread-2614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2614. 2024-11-13T00:16:22,858 DEBUG [Thread-2614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:22,858 DEBUG [Thread-2614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,858 DEBUG [Thread-2618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,858 DEBUG [Thread-2614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:22,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:22,991 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/191947360e974a96a878eef2ab4612c6 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/191947360e974a96a878eef2ab4612c6 2024-11-13T00:16:22,994 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/B of 041ae204889b5285cb06226a89ff8fa1 into 191947360e974a96a878eef2ab4612c6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:22,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:22,994 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/B, priority=13, startTime=1731456982566; duration=0sec 2024-11-13T00:16:22,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T00:16:22,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:B 2024-11-13T00:16:22,994 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T00:16:22,995 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T00:16:22,995 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1541): 041ae204889b5285cb06226a89ff8fa1/C is initiating minor compaction (all files) 2024-11-13T00:16:22,995 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 041ae204889b5285cb06226a89ff8fa1/C in TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:22,995 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/21a47cc410d34956999e3a72524af640, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01fbfb76c9fa400ba833ab659e199fcc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01e763f39e93493884e6a747bec15ae7] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp, totalSize=34.1 K 2024-11-13T00:16:22,995 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a47cc410d34956999e3a72524af640, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731456979323 2024-11-13T00:16:22,996 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 01fbfb76c9fa400ba833ab659e199fcc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731456979470 2024-11-13T00:16:22,996 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e763f39e93493884e6a747bec15ae7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731456981380 2024-11-13T00:16:22,996 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/bf4a5af1df194c4f98cf3c0098856af1 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/bf4a5af1df194c4f98cf3c0098856af1 2024-11-13T00:16:22,999 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/A of 041ae204889b5285cb06226a89ff8fa1 into bf4a5af1df194c4f98cf3c0098856af1(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:22,999 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:22,999 INFO [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/A, priority=13, startTime=1731456982565; duration=0sec 2024-11-13T00:16:22,999 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:22,999 DEBUG [RS:0;cb28580903fb:45273-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:A 2024-11-13T00:16:23,000 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 041ae204889b5285cb06226a89ff8fa1#C#compaction#556 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:16:23,001 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/a8d956baec464f35bfe6f00367c8511f is 50, key is test_row_0/C:col10/1731456981380/Put/seqid=0 2024-11-13T00:16:23,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742488_1664 (size=12881) 2024-11-13T00:16:23,407 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/a8d956baec464f35bfe6f00367c8511f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a8d956baec464f35bfe6f00367c8511f 2024-11-13T00:16:23,410 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 041ae204889b5285cb06226a89ff8fa1/C of 041ae204889b5285cb06226a89ff8fa1 into a8d956baec464f35bfe6f00367c8511f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:16:23,410 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:23,410 INFO [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., storeName=041ae204889b5285cb06226a89ff8fa1/C, priority=13, startTime=1731456982566; duration=0sec 2024-11-13T00:16:23,410 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T00:16:23,410 DEBUG [RS:0;cb28580903fb:45273-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 041ae204889b5285cb06226a89ff8fa1:C 2024-11-13T00:16:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45273 {}] regionserver.HRegion(8855): Flush requested on 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:23,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T00:16:23,447 INFO [Thread-2616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2616. 2024-11-13T00:16:23,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:23,447 DEBUG [Thread-2616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:23,447 DEBUG [Thread-2616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:23,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:23,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:23,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:23,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:23,448 DEBUG [Thread-2616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:23,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136ecc75d5727548fcba264c875cf3a4ea_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456982856/Put/seqid=0 2024-11-13T00:16:23,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742489_1665 (size=12454) 2024-11-13T00:16:23,507 INFO [Thread-2622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2622. 2024-11-13T00:16:23,507 DEBUG [Thread-2622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:23,507 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,507 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T00:16:23,626 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:23,647 INFO [Thread-2620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2620. 2024-11-13T00:16:23,647 DEBUG [Thread-2620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T00:16:23,647 DEBUG [Thread-2620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,647 DEBUG [Thread-2620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3180 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2928 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3154 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3188 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2951 2024-11-13T00:16:23,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T00:16:23,648 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:16:23,648 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:16:23,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:23,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:23,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T00:16:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T00:16:23,651 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456983651"}]},"ts":"1731456983651"} 2024-11-13T00:16:23,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T00:16:23,652 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T00:16:23,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T00:16:23,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, UNASSIGN}] 2024-11-13T00:16:23,654 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, UNASSIGN 2024-11-13T00:16:23,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=CLOSING, regionLocation=cb28580903fb,45273,1731456804594 2024-11-13T00:16:23,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, UNASSIGN because future has completed 2024-11-13T00:16:23,656 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:16:23,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594}] 2024-11-13T00:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T00:16:23,808 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:23,808 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:16:23,808 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 041ae204889b5285cb06226a89ff8fa1, disabling compactions & flushes 2024-11-13T00:16:23,808 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:23,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:23,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136ecc75d5727548fcba264c875cf3a4ea_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ecc75d5727548fcba264c875cf3a4ea_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:23,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/b5d949e95a1941d6a8b76a7b095b33ff, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:23,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/b5d949e95a1941d6a8b76a7b095b33ff is 175, key is test_row_0/A:col10/1731456982856/Put/seqid=0 2024-11-13T00:16:23,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742490_1666 (size=31255) 2024-11-13T00:16:23,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/b5d949e95a1941d6a8b76a7b095b33ff 2024-11-13T00:16:23,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/341a7c4acd3f435a81c568b81499c8b2 is 50, key is test_row_0/B:col10/1731456982856/Put/seqid=0 2024-11-13T00:16:23,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742491_1667 (size=12301) 2024-11-13T00:16:23,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/341a7c4acd3f435a81c568b81499c8b2 2024-11-13T00:16:23,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/8f2d25b4bd3c41d1af343f1db2484b8f is 50, key is test_row_0/C:col10/1731456982856/Put/seqid=0 2024-11-13T00:16:23,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742492_1668 (size=12301) 2024-11-13T00:16:23,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/8f2d25b4bd3c41d1af343f1db2484b8f 2024-11-13T00:16:23,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/b5d949e95a1941d6a8b76a7b095b33ff as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/b5d949e95a1941d6a8b76a7b095b33ff 2024-11-13T00:16:23,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/b5d949e95a1941d6a8b76a7b095b33ff, entries=150, sequenceid=443, filesize=30.5 K 2024-11-13T00:16:23,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/341a7c4acd3f435a81c568b81499c8b2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/341a7c4acd3f435a81c568b81499c8b2 2024-11-13T00:16:23,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/341a7c4acd3f435a81c568b81499c8b2, entries=150, sequenceid=443, filesize=12.0 K 2024-11-13T00:16:23,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/8f2d25b4bd3c41d1af343f1db2484b8f as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/8f2d25b4bd3c41d1af343f1db2484b8f 2024-11-13T00:16:23,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/8f2d25b4bd3c41d1af343f1db2484b8f, entries=150, sequenceid=443, filesize=12.0 K 2024-11-13T00:16:23,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 041ae204889b5285cb06226a89ff8fa1 in 462ms, sequenceid=443, compaction requested=false 2024-11-13T00:16:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 041ae204889b5285cb06226a89ff8fa1: 2024-11-13T00:16:23,909 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:23,909 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:23,909 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. after waiting 0 ms 2024-11-13T00:16:23,909 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:23,910 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 041ae204889b5285cb06226a89ff8fa1 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T00:16:23,910 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=A 2024-11-13T00:16:23,910 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:23,910 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=B 2024-11-13T00:16:23,910 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:23,910 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 041ae204889b5285cb06226a89ff8fa1, store=C 2024-11-13T00:16:23,910 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T00:16:23,914 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133e8668e995b14deca865bdda9b49241d_041ae204889b5285cb06226a89ff8fa1 is 50, key is test_row_0/A:col10/1731456983646/Put/seqid=0 2024-11-13T00:16:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742493_1669 (size=9914) 2024-11-13T00:16:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T00:16:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T00:16:24,319 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:16:24,321 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133e8668e995b14deca865bdda9b49241d_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133e8668e995b14deca865bdda9b49241d_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:24,322 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3a90ebdd794d49129d13ab674b6da72c, store: [table=TestAcidGuarantees family=A region=041ae204889b5285cb06226a89ff8fa1] 2024-11-13T00:16:24,323 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3a90ebdd794d49129d13ab674b6da72c is 175, key is test_row_0/A:col10/1731456983646/Put/seqid=0 2024-11-13T00:16:24,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742494_1670 (size=22561) 2024-11-13T00:16:24,726 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3a90ebdd794d49129d13ab674b6da72c 2024-11-13T00:16:24,734 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/73506b564a91420e9666a2e2951b0f8c is 50, key is test_row_0/B:col10/1731456983646/Put/seqid=0 2024-11-13T00:16:24,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742495_1671 (size=9857) 2024-11-13T00:16:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T00:16:25,148 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/73506b564a91420e9666a2e2951b0f8c 2024-11-13T00:16:25,153 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/db82378942764097af0b39c2a50f6e9b is 50, key is test_row_0/C:col10/1731456983646/Put/seqid=0 2024-11-13T00:16:25,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742496_1672 (size=9857) 2024-11-13T00:16:25,557 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/db82378942764097af0b39c2a50f6e9b 2024-11-13T00:16:25,560 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/A/3a90ebdd794d49129d13ab674b6da72c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3a90ebdd794d49129d13ab674b6da72c 2024-11-13T00:16:25,562 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3a90ebdd794d49129d13ab674b6da72c, entries=100, sequenceid=448, filesize=22.0 K 2024-11-13T00:16:25,563 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/B/73506b564a91420e9666a2e2951b0f8c as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/73506b564a91420e9666a2e2951b0f8c 2024-11-13T00:16:25,565 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/73506b564a91420e9666a2e2951b0f8c, entries=100, sequenceid=448, filesize=9.6 K 2024-11-13T00:16:25,566 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/.tmp/C/db82378942764097af0b39c2a50f6e9b as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/db82378942764097af0b39c2a50f6e9b 2024-11-13T00:16:25,568 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/db82378942764097af0b39c2a50f6e9b, entries=100, sequenceid=448, filesize=9.6 K 2024-11-13T00:16:25,568 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 041ae204889b5285cb06226a89ff8fa1 in 1659ms, sequenceid=448, compaction requested=true 2024-11-13T00:16:25,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/60e9256a49bd4b4a8aa468f19cf9d0ac, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/8660b196f9134cc39877e7ac2feaad4c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/5b66d296246944c7b1aff3dc6899811c, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54] to archive 2024-11-13T00:16:25,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:16:25,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/4f6a98d0a66c4f59b0976156ceeef3ec 2024-11-13T00:16:25,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/38f3237b883e4376a45c66c019187da4 2024-11-13T00:16:25,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/040b78b18e2648a480b621a3a3cfcf30 2024-11-13T00:16:25,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/60e9256a49bd4b4a8aa468f19cf9d0ac to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/60e9256a49bd4b4a8aa468f19cf9d0ac 2024-11-13T00:16:25,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6b872f93bc914fbebc3f7cb2d1b758fc 2024-11-13T00:16:25,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/6146a654d948467ba48da89bb6fff6b5 2024-11-13T00:16:25,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/7dfe2eac949a4727b9aa66f4b7a5bc5e 2024-11-13T00:16:25,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/8660b196f9134cc39877e7ac2feaad4c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/8660b196f9134cc39877e7ac2feaad4c 2024-11-13T00:16:25,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/243c1fda3812456a993cccce4d62ca25 2024-11-13T00:16:25,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/47fdb66e14554c60810a55a103d6b3a8 2024-11-13T00:16:25,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/50292e637dc64f9693aa72dadfb4f2cf 2024-11-13T00:16:25,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d7de3157c0d64b6792fc3d64cbff7787 2024-11-13T00:16:25,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/d18243476bf74d69a20848baa1005a0b 2024-11-13T00:16:25,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/440ce5d2ecd24868ba76264d51140202 2024-11-13T00:16:25,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/5b66d296246944c7b1aff3dc6899811c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/5b66d296246944c7b1aff3dc6899811c 2024-11-13T00:16:25,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3f915249ce1f4696af3b9c0617460af2 2024-11-13T00:16:25,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/649fd7f079514182ad8c36ec60b86d54 2024-11-13T00:16:25,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/aca9d3ff91754ad59920f1283c36d3b2, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/08fee221edb549b6a2378ed1db6d6e20, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca41a50727e84f6eb0746cbcceadb8aa, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/fb20f69a4a7a4e8494952de855815511, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/78d1b9b02e9749038fffa563dfb01b0d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/c17c0ee0d9c745e1989e5a2b7bcfffb0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca21b5a9f29844d88e198a5ad872958f, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/233b12fd490a4d0d8a154dd2a87e5ce9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5f9722d0ca3a4714b20de41699e3c3ca, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9e94f48152024c4396e856421640acd8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/2d5b3cc973b343f5bf65e9c9c1c88200, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/52cf3481a24040f2ad72d0b1044dd8f1, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5559ad5d04ed490f8f91a183e2dc9227, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/33d0b73ac0734c408a1706b08c9362b0, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9ec8316b3ad5434d9208bb30ab7d83be, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ef7f3a8990664fc39f79aa8c4d8da50e, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/079c93affb7f484cb519a82bd449bcb8] to archive 2024-11-13T00:16:25,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:16:25,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/aca9d3ff91754ad59920f1283c36d3b2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/aca9d3ff91754ad59920f1283c36d3b2 2024-11-13T00:16:25,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/08fee221edb549b6a2378ed1db6d6e20 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/08fee221edb549b6a2378ed1db6d6e20 2024-11-13T00:16:25,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca41a50727e84f6eb0746cbcceadb8aa to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca41a50727e84f6eb0746cbcceadb8aa 2024-11-13T00:16:25,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/fb20f69a4a7a4e8494952de855815511 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/fb20f69a4a7a4e8494952de855815511 2024-11-13T00:16:25,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/78d1b9b02e9749038fffa563dfb01b0d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/78d1b9b02e9749038fffa563dfb01b0d 2024-11-13T00:16:25,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/c17c0ee0d9c745e1989e5a2b7bcfffb0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/c17c0ee0d9c745e1989e5a2b7bcfffb0 2024-11-13T00:16:25,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca21b5a9f29844d88e198a5ad872958f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ca21b5a9f29844d88e198a5ad872958f 2024-11-13T00:16:25,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/233b12fd490a4d0d8a154dd2a87e5ce9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/233b12fd490a4d0d8a154dd2a87e5ce9 2024-11-13T00:16:25,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5f9722d0ca3a4714b20de41699e3c3ca to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5f9722d0ca3a4714b20de41699e3c3ca 2024-11-13T00:16:25,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9e94f48152024c4396e856421640acd8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9e94f48152024c4396e856421640acd8 2024-11-13T00:16:25,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/2d5b3cc973b343f5bf65e9c9c1c88200 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/2d5b3cc973b343f5bf65e9c9c1c88200 2024-11-13T00:16:25,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/52cf3481a24040f2ad72d0b1044dd8f1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/52cf3481a24040f2ad72d0b1044dd8f1 2024-11-13T00:16:25,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5559ad5d04ed490f8f91a183e2dc9227 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/5559ad5d04ed490f8f91a183e2dc9227 2024-11-13T00:16:25,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/33d0b73ac0734c408a1706b08c9362b0 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/33d0b73ac0734c408a1706b08c9362b0 2024-11-13T00:16:25,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9ec8316b3ad5434d9208bb30ab7d83be to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/9ec8316b3ad5434d9208bb30ab7d83be 2024-11-13T00:16:25,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ef7f3a8990664fc39f79aa8c4d8da50e to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/ef7f3a8990664fc39f79aa8c4d8da50e 2024-11-13T00:16:25,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/079c93affb7f484cb519a82bd449bcb8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/079c93affb7f484cb519a82bd449bcb8 2024-11-13T00:16:25,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/06637d47c4fb4032a7f7f83791b6b9f8, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a57297ed594e46da81a91d6735e50ed5, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/aa25fcbeef044b9fbc9777a9249209c7, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/350f67f32fc546cbbcc5e1059eb415fe, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/73e83edc215f4ecf9a7a8a8fe3cce945, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7be1e168b4254743a86f73558b7de999, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/9d520e18bbf6473397db6c6ae55002e9, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6dc779d3369e45408647f27272e15105, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/10b87bf8d95e4a468f9174c4cf5cd450, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6666186f4611414d94dfec84033ef0ee, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7725e1e13e6141a7a0a3d08e134eec9d, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/65362726e5764130abf312a4910e5f4b, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7ba92dd7c39e4be3a11ab2f91b4c8967, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/21a47cc410d34956999e3a72524af640, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/34cfe164914c4fe7b6e00b2738fa2fbb, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01fbfb76c9fa400ba833ab659e199fcc, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01e763f39e93493884e6a747bec15ae7] to archive 2024-11-13T00:16:25,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:16:25,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/06637d47c4fb4032a7f7f83791b6b9f8 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/06637d47c4fb4032a7f7f83791b6b9f8 2024-11-13T00:16:25,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a57297ed594e46da81a91d6735e50ed5 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a57297ed594e46da81a91d6735e50ed5 2024-11-13T00:16:25,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/aa25fcbeef044b9fbc9777a9249209c7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/aa25fcbeef044b9fbc9777a9249209c7 2024-11-13T00:16:25,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/350f67f32fc546cbbcc5e1059eb415fe to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/350f67f32fc546cbbcc5e1059eb415fe 2024-11-13T00:16:25,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/73e83edc215f4ecf9a7a8a8fe3cce945 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/73e83edc215f4ecf9a7a8a8fe3cce945 2024-11-13T00:16:25,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7be1e168b4254743a86f73558b7de999 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7be1e168b4254743a86f73558b7de999 2024-11-13T00:16:25,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/9d520e18bbf6473397db6c6ae55002e9 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/9d520e18bbf6473397db6c6ae55002e9 2024-11-13T00:16:25,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6dc779d3369e45408647f27272e15105 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6dc779d3369e45408647f27272e15105 2024-11-13T00:16:25,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/10b87bf8d95e4a468f9174c4cf5cd450 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/10b87bf8d95e4a468f9174c4cf5cd450 2024-11-13T00:16:25,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6666186f4611414d94dfec84033ef0ee to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/6666186f4611414d94dfec84033ef0ee 2024-11-13T00:16:25,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7725e1e13e6141a7a0a3d08e134eec9d to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7725e1e13e6141a7a0a3d08e134eec9d 2024-11-13T00:16:25,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/65362726e5764130abf312a4910e5f4b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/65362726e5764130abf312a4910e5f4b 2024-11-13T00:16:25,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7ba92dd7c39e4be3a11ab2f91b4c8967 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/7ba92dd7c39e4be3a11ab2f91b4c8967 2024-11-13T00:16:25,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/21a47cc410d34956999e3a72524af640 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/21a47cc410d34956999e3a72524af640 2024-11-13T00:16:25,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/34cfe164914c4fe7b6e00b2738fa2fbb to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/34cfe164914c4fe7b6e00b2738fa2fbb 2024-11-13T00:16:25,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01fbfb76c9fa400ba833ab659e199fcc to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01fbfb76c9fa400ba833ab659e199fcc 2024-11-13T00:16:25,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01e763f39e93493884e6a747bec15ae7 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/01e763f39e93493884e6a747bec15ae7 2024-11-13T00:16:25,613 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/recovered.edits/451.seqid, newMaxSeqId=451, maxSeqId=4 2024-11-13T00:16:25,613 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. 2024-11-13T00:16:25,613 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 041ae204889b5285cb06226a89ff8fa1: Waiting for close lock at 1731456983808Running coprocessor pre-close hooks at 1731456983808Disabling compacts and flushes for region at 1731456983808Disabling writes for close at 1731456983909 (+101 ms)Obtaining lock to block concurrent updates at 1731456983910 (+1 ms)Preparing flush snapshotting stores in 041ae204889b5285cb06226a89ff8fa1 at 1731456983910Finished memstore snapshotting TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731456983910Flushing stores of TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1. at 1731456983910Flushing 041ae204889b5285cb06226a89ff8fa1/A: creating writer at 1731456983910Flushing mob file 041ae204889b5285cb06226a89ff8fa1/A: appending metadata at 1731456983914 (+4 ms)Flushing mob file 041ae204889b5285cb06226a89ff8fa1/A: closing flushed file at 1731456983914Flushing 041ae204889b5285cb06226a89ff8fa1/A: appending metadata at 1731456984322 (+408 ms)Flushing 041ae204889b5285cb06226a89ff8fa1/A: closing flushed file at 1731456984322Flushing 041ae204889b5285cb06226a89ff8fa1/B: creating writer at 1731456984730 (+408 ms)Flushing 041ae204889b5285cb06226a89ff8fa1/B: appending metadata at 1731456984733 (+3 ms)Flushing 041ae204889b5285cb06226a89ff8fa1/B: closing flushed file at 1731456984733Flushing 041ae204889b5285cb06226a89ff8fa1/C: creating writer at 1731456985151 (+418 ms)Flushing 041ae204889b5285cb06226a89ff8fa1/C: appending metadata at 1731456985153 (+2 ms)Flushing 041ae204889b5285cb06226a89ff8fa1/C: closing flushed file at 1731456985153Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2246fb1: reopening flushed file at 1731456985559 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29bc1e19: reopening flushed file at 1731456985563 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bb00485: reopening flushed file at 1731456985565 (+2 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 041ae204889b5285cb06226a89ff8fa1 in 1659ms, sequenceid=448, compaction requested=true at 1731456985569 (+4 ms)Writing region close event to WAL at 1731456985610 (+41 ms)Running coprocessor post-close hooks at 1731456985613 (+3 ms)Closed at 1731456985613 2024-11-13T00:16:25,615 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,615 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=041ae204889b5285cb06226a89ff8fa1, regionState=CLOSED 2024-11-13T00:16:25,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 because future has completed 2024-11-13T00:16:25,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-13T00:16:25,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 041ae204889b5285cb06226a89ff8fa1, server=cb28580903fb,45273,1731456804594 in 1.9610 sec 2024-11-13T00:16:25,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-13T00:16:25,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=041ae204889b5285cb06226a89ff8fa1, UNASSIGN in 1.9650 sec 2024-11-13T00:16:25,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-13T00:16:25,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-13T00:16:25,621 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731456985621"}]},"ts":"1731456985621"} 2024-11-13T00:16:25,622 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T00:16:25,622 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T00:16:25,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9750 sec 2024-11-13T00:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T00:16:25,797 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:25,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T00:16:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:25,798 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-13T00:16:25,799 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:25,801 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,802 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C, FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/recovered.edits] 2024-11-13T00:16:25,804 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3a90ebdd794d49129d13ab674b6da72c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/3a90ebdd794d49129d13ab674b6da72c 2024-11-13T00:16:25,805 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/b5d949e95a1941d6a8b76a7b095b33ff to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/b5d949e95a1941d6a8b76a7b095b33ff 2024-11-13T00:16:25,806 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/bf4a5af1df194c4f98cf3c0098856af1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/A/bf4a5af1df194c4f98cf3c0098856af1 2024-11-13T00:16:25,808 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/191947360e974a96a878eef2ab4612c6 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/191947360e974a96a878eef2ab4612c6 2024-11-13T00:16:25,808 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/341a7c4acd3f435a81c568b81499c8b2 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/341a7c4acd3f435a81c568b81499c8b2 2024-11-13T00:16:25,809 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/73506b564a91420e9666a2e2951b0f8c to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/B/73506b564a91420e9666a2e2951b0f8c 2024-11-13T00:16:25,811 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/8f2d25b4bd3c41d1af343f1db2484b8f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/8f2d25b4bd3c41d1af343f1db2484b8f 2024-11-13T00:16:25,812 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a8d956baec464f35bfe6f00367c8511f to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/a8d956baec464f35bfe6f00367c8511f 2024-11-13T00:16:25,812 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/db82378942764097af0b39c2a50f6e9b to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/C/db82378942764097af0b39c2a50f6e9b 2024-11-13T00:16:25,814 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/recovered.edits/451.seqid to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1/recovered.edits/451.seqid 2024-11-13T00:16:25,815 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/default/TestAcidGuarantees/041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,815 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T00:16:25,815 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T00:16:25,816 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T00:16:25,818 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111330223f9620ca4ec8aac5be55faea2cb0_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111330223f9620ca4ec8aac5be55faea2cb0_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,818 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133bf07e0ee45c4451abf0b382c057296a_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133bf07e0ee45c4451abf0b382c057296a_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,819 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133e8668e995b14deca865bdda9b49241d_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133e8668e995b14deca865bdda9b49241d_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,820 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134dcd98df38f2413dbf1dd102d721384a_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134dcd98df38f2413dbf1dd102d721384a_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,820 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135cfa07d27c2d446a89464a7ca8b9dd69_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135cfa07d27c2d446a89464a7ca8b9dd69_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,821 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136047afdca72f41828fe01674a4f850f2_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136047afdca72f41828fe01674a4f850f2_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,822 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366a6d8d4fa7c4d969dd5aecd0f845474_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366a6d8d4fa7c4d969dd5aecd0f845474_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,823 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ecc75d5727548fcba264c875cf3a4ea_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136ecc75d5727548fcba264c875cf3a4ea_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,823 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f2951df6753494f86a948b1db827a0f_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f2951df6753494f86a948b1db827a0f_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,824 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371d27564393a4f01b0d835aead1ce3eb_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371d27564393a4f01b0d835aead1ce3eb_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,825 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137233d25f570343048fb69a6eef57481d_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137233d25f570343048fb69a6eef57481d_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,826 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383239b5ee13c4cb5b1e67c33c1c4eabb_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383239b5ee13c4cb5b1e67c33c1c4eabb_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,826 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139a44acf707374ad18a7b893b300f3ea4_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139a44acf707374ad18a7b893b300f3ea4_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,827 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139cd905f138254fcc9e3467943eb14bd3_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139cd905f138254fcc9e3467943eb14bd3_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,828 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4cd83e4eac343078623b122b993fa1e_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4cd83e4eac343078623b122b993fa1e_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,828 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2414e1d36ce4e65a82b3eb7903c6a3c_041ae204889b5285cb06226a89ff8fa1 to hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2414e1d36ce4e65a82b3eb7903c6a3c_041ae204889b5285cb06226a89ff8fa1 2024-11-13T00:16:25,829 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T00:16:25,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:25,833 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T00:16:25,834 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T00:16:25,835 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:25,835 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T00:16:25,835 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731456985835"}]},"ts":"9223372036854775807"} 2024-11-13T00:16:25,837 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T00:16:25,837 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 041ae204889b5285cb06226a89ff8fa1, NAME => 'TestAcidGuarantees,,1731456959684.041ae204889b5285cb06226a89ff8fa1.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T00:16:25,837 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T00:16:25,837 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731456985837"}]},"ts":"9223372036854775807"} 2024-11-13T00:16:25,838 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T00:16:25,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T00:16:25,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-13T00:16:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-13T00:16:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T00:16:25,906 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T00:16:25,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 249) - Thread LEAK? -, OpenFileDescriptor=453 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 401), ProcessCount=11 (was 11), AvailableMemoryMB=692 (was 783) 2024-11-13T00:16:25,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T00:16:25,916 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:16:25,916 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:16:25,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:25,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:25,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:16:25,916 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T00:16:25,916 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2051383393, stopped=false 2024-11-13T00:16:25,916 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cb28580903fb,41869,1731456803739 2024-11-13T00:16:25,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T00:16:25,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T00:16:25,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:16:25,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:16:25,918 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T00:16:25,919 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:16:25,919 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:16:25,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:25,919 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cb28580903fb,45273,1731456804594' ***** 2024-11-13T00:16:25,919 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T00:16:25,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:16:25,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:16:25,919 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T00:16:25,920 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T00:16:25,920 INFO [RS:0;cb28580903fb:45273 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T00:16:25,920 INFO [RS:0;cb28580903fb:45273 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T00:16:25,920 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(959): stopping server cb28580903fb,45273,1731456804594 2024-11-13T00:16:25,920 INFO [RS:0;cb28580903fb:45273 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T00:16:25,920 INFO [RS:0;cb28580903fb:45273 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cb28580903fb:45273. 2024-11-13T00:16:25,920 DEBUG [RS:0;cb28580903fb:45273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:16:25,920 DEBUG [RS:0;cb28580903fb:45273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:25,921 INFO [RS:0;cb28580903fb:45273 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T00:16:25,921 INFO [RS:0;cb28580903fb:45273 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T00:16:25,921 INFO [RS:0;cb28580903fb:45273 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T00:16:25,921 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T00:16:25,921 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T00:16:25,921 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T00:16:25,922 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T00:16:25,922 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:25,922 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T00:16:25,922 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T00:16:25,922 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T00:16:25,922 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T00:16:25,922 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T00:16:25,945 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/info/27ba2a8ea78a4dab8168817e35860a81 is 95, key is TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1./info:/1731456842683/DeleteFamily/seqid=0 2024-11-13T00:16:25,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742497_1673 (size=6514) 2024-11-13T00:16:25,962 INFO [regionserver/cb28580903fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:16:26,034 INFO [regionserver/cb28580903fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T00:16:26,034 INFO [regionserver/cb28580903fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T00:16:26,122 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:26,322 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:26,349 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/info/27ba2a8ea78a4dab8168817e35860a81 2024-11-13T00:16:26,366 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/ns/40888a45fd184d3d908b39f5bfa7c951 is 93, key is TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1./ns:/1731456842630/DeleteFamily/seqid=0 2024-11-13T00:16:26,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742498_1674 (size=5906) 2024-11-13T00:16:26,522 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:26,722 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:26,770 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/ns/40888a45fd184d3d908b39f5bfa7c951 2024-11-13T00:16:26,787 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/rep_barrier/b4ffb5418ce24c8a81ae7c8584f251d5 is 102, key is TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1./rep_barrier:/1731456842630/DeleteFamily/seqid=0 2024-11-13T00:16:26,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742499_1675 (size=6025) 2024-11-13T00:16:26,923 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:27,001 INFO [regionserver/cb28580903fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-13T00:16:27,123 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:27,190 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/rep_barrier/b4ffb5418ce24c8a81ae7c8584f251d5 2024-11-13T00:16:27,207 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/table/3bed4410d28f40bf9cbf811ed99f4f9d is 96, key is TestAcidGuarantees,,1731456808508.883bdd7db86faeb1b8a9722c1fd955c1./table:/1731456842630/DeleteFamily/seqid=0 2024-11-13T00:16:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742500_1676 (size=5936) 2024-11-13T00:16:27,323 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:27,523 DEBUG [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T00:16:27,611 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/table/3bed4410d28f40bf9cbf811ed99f4f9d 2024-11-13T00:16:27,614 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/info/27ba2a8ea78a4dab8168817e35860a81 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/info/27ba2a8ea78a4dab8168817e35860a81 2024-11-13T00:16:27,617 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/info/27ba2a8ea78a4dab8168817e35860a81, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T00:16:27,617 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/ns/40888a45fd184d3d908b39f5bfa7c951 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/ns/40888a45fd184d3d908b39f5bfa7c951 2024-11-13T00:16:27,620 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/ns/40888a45fd184d3d908b39f5bfa7c951, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T00:16:27,620 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/rep_barrier/b4ffb5418ce24c8a81ae7c8584f251d5 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/rep_barrier/b4ffb5418ce24c8a81ae7c8584f251d5 2024-11-13T00:16:27,622 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/rep_barrier/b4ffb5418ce24c8a81ae7c8584f251d5, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T00:16:27,623 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/.tmp/table/3bed4410d28f40bf9cbf811ed99f4f9d as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/table/3bed4410d28f40bf9cbf811ed99f4f9d 2024-11-13T00:16:27,625 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/table/3bed4410d28f40bf9cbf811ed99f4f9d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T00:16:27,626 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-11-13T00:16:27,629 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T00:16:27,630 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T00:16:27,630 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T00:16:27,630 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731456985921Running coprocessor pre-close hooks at 1731456985922 (+1 ms)Disabling compacts and flushes for region at 1731456985922Disabling writes for close at 1731456985922Obtaining lock to block concurrent updates at 1731456985922Preparing flush snapshotting stores in 1588230740 at 1731456985922Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731456985922Flushing stores of hbase:meta,,1.1588230740 at 1731456985923 (+1 ms)Flushing 1588230740/info: creating writer at 1731456985923Flushing 1588230740/info: appending metadata at 1731456985945 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731456985945Flushing 1588230740/ns: creating writer at 1731456986352 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731456986366 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731456986366Flushing 1588230740/rep_barrier: creating writer at 1731456986773 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731456986787 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731456986787Flushing 1588230740/table: creating writer at 1731456987193 (+406 ms)Flushing 1588230740/table: appending metadata at 1731456987207 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731456987207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55137880: reopening flushed file at 1731456987613 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15c34da8: reopening flushed file at 1731456987617 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a985259: reopening flushed file at 1731456987620 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@513c791c: reopening flushed file at 1731456987623 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1731456987626 (+3 ms)Writing region close event to WAL at 1731456987627 (+1 ms)Running coprocessor post-close hooks at 1731456987630 (+3 ms)Closed at 1731456987630 2024-11-13T00:16:27,630 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T00:16:27,724 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(976): stopping server cb28580903fb,45273,1731456804594; all regions closed. 2024-11-13T00:16:27,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741834_1010 (size=25344) 2024-11-13T00:16:27,730 DEBUG [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/oldWALs 2024-11-13T00:16:27,730 INFO [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb28580903fb%2C45273%2C1731456804594.meta:.meta(num 1731456808101) 2024-11-13T00:16:27,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741832_1008 (size=19019686) 2024-11-13T00:16:27,732 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/WALs/cb28580903fb,45273,1731456804594/cb28580903fb%2C45273%2C1731456804594.1731456807136 not finished, retry = 0 2024-11-13T00:16:27,834 DEBUG [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/oldWALs 2024-11-13T00:16:27,834 INFO [RS:0;cb28580903fb:45273 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb28580903fb%2C45273%2C1731456804594:(num 1731456807136) 2024-11-13T00:16:27,834 DEBUG [RS:0;cb28580903fb:45273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:16:27,834 INFO [RS:0;cb28580903fb:45273 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:16:27,835 INFO [RS:0;cb28580903fb:45273 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T00:16:27,835 INFO [RS:0;cb28580903fb:45273 {}] hbase.ChoreService(370): Chore service for: regionserver/cb28580903fb:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T00:16:27,835 INFO [RS:0;cb28580903fb:45273 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T00:16:27,835 INFO [regionserver/cb28580903fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T00:16:27,836 INFO [RS:0;cb28580903fb:45273 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45273 2024-11-13T00:16:27,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cb28580903fb,45273,1731456804594 2024-11-13T00:16:27,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T00:16:27,840 INFO [RS:0;cb28580903fb:45273 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T00:16:27,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cb28580903fb,45273,1731456804594] 2024-11-13T00:16:27,844 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cb28580903fb,45273,1731456804594 already deleted, retry=false 2024-11-13T00:16:27,844 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cb28580903fb,45273,1731456804594 expired; onlineServers=0 2024-11-13T00:16:27,845 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cb28580903fb,41869,1731456803739' ***** 2024-11-13T00:16:27,845 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T00:16:27,845 INFO [M:0;cb28580903fb:41869 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T00:16:27,845 INFO [M:0;cb28580903fb:41869 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T00:16:27,845 DEBUG [M:0;cb28580903fb:41869 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T00:16:27,845 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T00:16:27,845 DEBUG [M:0;cb28580903fb:41869 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T00:16:27,845 DEBUG [master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.large.0-1731456806847 {}] cleaner.HFileCleaner(306): Exit Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.large.0-1731456806847,5,FailOnTimeoutGroup] 2024-11-13T00:16:27,845 DEBUG [master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.small.0-1731456806847 {}] cleaner.HFileCleaner(306): Exit Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.small.0-1731456806847,5,FailOnTimeoutGroup] 2024-11-13T00:16:27,845 INFO [M:0;cb28580903fb:41869 {}] hbase.ChoreService(370): Chore service for: master/cb28580903fb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T00:16:27,845 INFO [M:0;cb28580903fb:41869 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T00:16:27,845 DEBUG [M:0;cb28580903fb:41869 {}] master.HMaster(1795): Stopping service threads 2024-11-13T00:16:27,845 INFO [M:0;cb28580903fb:41869 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T00:16:27,845 INFO [M:0;cb28580903fb:41869 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T00:16:27,846 ERROR [M:0;cb28580903fb:41869 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T00:16:27,846 INFO [M:0;cb28580903fb:41869 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T00:16:27,846 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T00:16:27,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T00:16:27,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:16:27,847 DEBUG [M:0;cb28580903fb:41869 {}] zookeeper.ZKUtil(347): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T00:16:27,847 WARN [M:0;cb28580903fb:41869 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T00:16:27,848 INFO [M:0;cb28580903fb:41869 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/.lastflushedseqids 2024-11-13T00:16:27,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742501_1677 (size=72) 2024-11-13T00:16:27,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:16:27,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45273-0x1010eac0ace0001, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:16:27,943 INFO [RS:0;cb28580903fb:45273 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T00:16:27,943 INFO [RS:0;cb28580903fb:45273 {}] regionserver.HRegionServer(1031): Exiting; stopping=cb28580903fb,45273,1731456804594; zookeeper connection closed. 2024-11-13T00:16:27,943 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@751cb443 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@751cb443 2024-11-13T00:16:27,944 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T00:16:28,257 INFO [M:0;cb28580903fb:41869 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T00:16:28,257 INFO [M:0;cb28580903fb:41869 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T00:16:28,257 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T00:16:28,258 INFO [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:16:28,258 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:16:28,258 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T00:16:28,258 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:16:28,258 INFO [M:0;cb28580903fb:41869 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=787.12 KB heapSize=964.31 KB 2024-11-13T00:16:28,273 DEBUG [M:0;cb28580903fb:41869 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/575ac4915ec24c62b9ad8f4381a094b2 is 82, key is hbase:meta,,1/info:regioninfo/1731456808186/Put/seqid=0 2024-11-13T00:16:28,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742502_1678 (size=5672) 2024-11-13T00:16:28,677 INFO [M:0;cb28580903fb:41869 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/575ac4915ec24c62b9ad8f4381a094b2 2024-11-13T00:16:28,699 DEBUG [M:0;cb28580903fb:41869 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b26926184f814a40a097c77de467faba is 2479, key is \x00\x00\x00\x00\x00\x00\x00^/proc:d/1731456905430/Put/seqid=0 2024-11-13T00:16:28,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742503_1679 (size=46069) 2024-11-13T00:16:28,703 INFO [M:0;cb28580903fb:41869 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=786.57 KB at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b26926184f814a40a097c77de467faba 2024-11-13T00:16:28,706 INFO [M:0;cb28580903fb:41869 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b26926184f814a40a097c77de467faba 2024-11-13T00:16:28,720 DEBUG [M:0;cb28580903fb:41869 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3777eaaef72248ce93fe35d4b4b3c555 is 69, key is cb28580903fb,45273,1731456804594/rs:state/1731456806902/Put/seqid=0 2024-11-13T00:16:28,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073742504_1680 (size=5156) 2024-11-13T00:16:29,124 INFO [M:0;cb28580903fb:41869 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3777eaaef72248ce93fe35d4b4b3c555 2024-11-13T00:16:29,127 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/575ac4915ec24c62b9ad8f4381a094b2 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/575ac4915ec24c62b9ad8f4381a094b2 2024-11-13T00:16:29,130 INFO [M:0;cb28580903fb:41869 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/575ac4915ec24c62b9ad8f4381a094b2, entries=8, sequenceid=2186, filesize=5.5 K 2024-11-13T00:16:29,130 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b26926184f814a40a097c77de467faba as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b26926184f814a40a097c77de467faba 2024-11-13T00:16:29,132 INFO [M:0;cb28580903fb:41869 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b26926184f814a40a097c77de467faba 2024-11-13T00:16:29,133 INFO [M:0;cb28580903fb:41869 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b26926184f814a40a097c77de467faba, entries=174, sequenceid=2186, filesize=45.0 K 2024-11-13T00:16:29,133 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3777eaaef72248ce93fe35d4b4b3c555 as hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3777eaaef72248ce93fe35d4b4b3c555 2024-11-13T00:16:29,135 INFO [M:0;cb28580903fb:41869 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/35bdcf63-e4ea-e9b4-89e8-36e67a407c54/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3777eaaef72248ce93fe35d4b4b3c555, entries=1, sequenceid=2186, filesize=5.0 K 2024-11-13T00:16:29,136 INFO [M:0;cb28580903fb:41869 {}] regionserver.HRegion(3140): Finished flush of dataSize ~787.12 KB/806012, heapSize ~964.02 KB/987152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 878ms, sequenceid=2186, compaction requested=false 2024-11-13T00:16:29,137 INFO [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:16:29,137 DEBUG [M:0;cb28580903fb:41869 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731456988257Disabling compacts and flushes for region at 1731456988257Disabling writes for close at 1731456988258 (+1 ms)Obtaining lock to block concurrent updates at 1731456988258Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731456988258Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=806012, getHeapSize=987392, getOffHeapSize=0, getCellsCount=2385 at 1731456988258Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731456988259 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731456988259Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731456988273 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731456988273Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731456988680 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731456988698 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731456988698Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731456988706 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731456988720 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731456988720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@624d1aa8: reopening flushed file at 1731456989127 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75f43cbd: reopening flushed file at 1731456989130 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f87ede4: reopening flushed file at 1731456989133 (+3 ms)Finished flush of dataSize ~787.12 KB/806012, heapSize ~964.02 KB/987152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 878ms, sequenceid=2186, compaction requested=false at 1731456989136 (+3 ms)Writing region close event to WAL at 1731456989137 (+1 ms)Closed at 1731456989137 2024-11-13T00:16:29,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741830_1006 (size=949317) 2024-11-13T00:16:29,140 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T00:16:29,140 INFO [M:0;cb28580903fb:41869 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T00:16:29,140 INFO [M:0;cb28580903fb:41869 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41869 2024-11-13T00:16:29,140 INFO [M:0;cb28580903fb:41869 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T00:16:29,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:16:29,243 INFO [M:0;cb28580903fb:41869 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T00:16:29,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41869-0x1010eac0ace0000, quorum=127.0.0.1:55432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:16:29,247 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:16:29,250 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T00:16:29,250 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T00:16:29,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T00:16:29,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/hadoop.log.dir/,STOPPED} 2024-11-13T00:16:29,253 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T00:16:29,253 WARN [BP-1378608587-172.17.0.2-1731456800440 heartbeating to localhost/127.0.0.1:45809 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T00:16:29,253 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T00:16:29,253 WARN [BP-1378608587-172.17.0.2-1731456800440 heartbeating to localhost/127.0.0.1:45809 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1378608587-172.17.0.2-1731456800440 (Datanode Uuid 01441b4e-4e4a-4f75-ac2b-c3de0838f229) service to localhost/127.0.0.1:45809 2024-11-13T00:16:29,256 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/data/data1/current/BP-1378608587-172.17.0.2-1731456800440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:16:29,256 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/cluster_3f4e2543-7d7b-c4b4-fd96-9aac157d6948/data/data2/current/BP-1378608587-172.17.0.2-1731456800440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:16:29,257 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T00:16:29,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T00:16:29,265 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T00:16:29,265 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T00:16:29,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T00:16:29,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/04068495-c5af-8d30-e50d-0b2897e3e0de/hadoop.log.dir/,STOPPED} 2024-11-13T00:16:29,287 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T00:16:29,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down